diff --git a/Cargo.lock b/Cargo.lock index d122275159..210ba07017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,6 +1631,7 @@ dependencies = [ "litebox_hal", "litebox_platform_vm_kernel", "log", + "spin 0.10.0", ] [[package]] @@ -1846,6 +1847,7 @@ dependencies = [ "hashbrown", "litebox", "litebox_common_linux", + "litebox_hal", "litebox_platform", "litebox_util_log", "spin 0.10.0", @@ -2032,6 +2034,7 @@ dependencies = [ "aes", "litebox", "litebox_broker_local_vm_userland", + "litebox_broker_protocol", "litebox_common_linux", "litebox_common_optee", "litebox_common_vm_abi", @@ -2074,6 +2077,7 @@ dependencies = [ "log", "serde", "serde_json", + "spin 0.10.0", "tar-no-std", "zerocopy", ] diff --git a/dev_tools/qemu_test_lib.sh b/dev_tools/qemu_test_lib.sh index a4f52d8d4a..08db8332e9 100644 --- a/dev_tools/qemu_test_lib.sh +++ b/dev_tools/qemu_test_lib.sh @@ -4,16 +4,18 @@ # Licensed under the MIT license. # Shared by the dev_tools/run_optee_on_*.sh scripts; source it. Reads QEMU, -# QEMU_ACCEL, TIMEOUT, LITEBOX_LOG and CARGO_BUILD (see the scripts), makes a -# relative CARGO_TARGET_DIR absolute, and creates $WORK, removed on exit. +# QEMU_ACCEL, QEMU_CPU, TIMEOUT, LITEBOX_LOG and CARGO_BUILD (see the scripts), +# makes a relative CARGO_TARGET_DIR absolute, and creates $WORK, removed on +# exit. QEMU=${QEMU:-qemu-system-x86_64} if [[ -z ${QEMU_ACCEL:-} ]]; then if [[ -r /dev/kvm && -w /dev/kvm ]]; then QEMU_ACCEL=kvm; else QEMU_ACCEL=tcg; fi fi case $QEMU_ACCEL in - kvm) CPU=host ;; - tcg) CPU=max ;; + # invtsc (not migratable) lets QEMU publish the TSC frequency to the guest. + kvm) CPU=${QEMU_CPU:-host,+invtsc} ;; + tcg) CPU=${QEMU_CPU:-max} ;; *) echo "error: unknown QEMU_ACCEL '$QEMU_ACCEL'" >&2; exit 2 ;; esac TIMEOUT=${TIMEOUT:-120} @@ -52,10 +54,13 @@ pass=0 fail=0 failed=() -# run_vm NAME KERNEL PAYLOAD: boots KERNEL with PAYLOAD as its initrd and -# records the outcome; with $verbose set, prints passing logs too. +# run_vm NAME KERNEL PAYLOAD [QEMU_ARG]...: boots KERNEL with PAYLOAD as its +# initrd and records the outcome; with $verbose set, prints passing logs too. +# If $check names a function, a guest pass also needs `$check NAME` to +# succeed; it prints why not otherwise. run_vm() { local name=$1 kernel=$2 payload=$3 log="$WORK/$1.log" status why + shift 3 set +e # shellcheck disable=SC2086 # QEMU is a command line timeout "$TIMEOUT" $QEMU \ @@ -64,16 +69,18 @@ run_vm() { -kernel "$kernel" -initrd "$payload" -append "litebox.log=${LITEBOX_LOG:-info}" \ -serial stdio -display none -no-reboot \ -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ + "$@" \ "$log" 2>&1 status=$? set -e # isa-debug-exit: QEMU exits with (value << 1) | 1; 33 = pass. - if [[ $status -eq 33 ]]; then + if [[ $status -eq 33 ]] && { [[ -z ${check:-} ]] || why=$($check "$name"); }; then echo "PASS $name" pass=$((pass + 1)) [[ ${verbose:-0} -eq 0 ]] || cat "$log" else case $status in + 33) ;; 65) why="guest reported failure" ;; 124) why="timed out after ${TIMEOUT}s" ;; *) why="exit status $status" ;; diff --git a/dev_tools/run_optee_on_qemu.sh b/dev_tools/run_optee_on_qemu.sh index 5e66f6a360..40fb8aa031 100755 --- a/dev_tools/run_optee_on_qemu.sh +++ b/dev_tools/run_optee_on_qemu.sh @@ -17,6 +17,7 @@ # Environment: # QEMU QEMU binary/command (default: qemu-system-x86_64) # QEMU_ACCEL kvm or tcg (default: kvm if /dev/kvm is usable, else tcg) +# QEMU_CPU QEMU CPU model (default: host,+invtsc for kvm, max for tcg) # TIMEOUT per-run timeout in seconds (default: 120) # LITEBOX_LOG guest log level, e.g. debug (default: info) # CARGO_BUILD command that builds the runner, run in its directory diff --git a/dev_tools/run_optee_on_vm_userland.sh b/dev_tools/run_optee_on_vm_userland.sh index 60279d50e9..4219de4c56 100755 --- a/dev_tools/run_optee_on_vm_userland.sh +++ b/dev_tools/run_optee_on_vm_userland.sh @@ -14,17 +14,22 @@ # open_session commands name the TAs it needs in their "ta" fields, or a TA # with a *-cmds.json of its own. # +# Runner processes' standard streams are a virtio console, backed by files: +# TA output must reach it, and in debug builds, whose runners echo standard +# input, so must the line fed to it. +# # Usage: dev_tools/run_optee_on_vm_userland.sh [-t ]... [-u] [-r] [-v] # -t test to run, e.g. ta-death or hello-ta; append @default to a TA to # omit its cmds.json (repeatable; default: every scenario, plus # hello-ta@default for the default-commands path) # -u use unmodified ldelf and TAs; the kernel reflects their syscalls # -r build and run the release kernel -# -v print the full guest log for passing runs too +# -v print the full guest log and console output for passing runs too # # Environment: # QEMU QEMU binary/command (default: qemu-system-x86_64) # QEMU_ACCEL kvm or tcg (default: kvm if /dev/kvm is usable, else tcg) +# QEMU_CPU QEMU CPU model (default: host,+invtsc for kvm, max for tcg) # TIMEOUT per-run timeout in seconds (default: 120) # LITEBOX_LOG guest log level, e.g. debug (default: info) # CARGO_BUILD command that builds the runners (and the rewriter unless @@ -93,6 +98,33 @@ prepare() { fi } +STDIN_LINE="hello from the host" + +# The virtio console's files for test $1. +console_args() { + printf '%s\n' "$STDIN_LINE" >"$WORK/$1.stdin" + : >"$WORK/$1.stdout" + echo -device virtio-serial-pci,id=vs0,disable-legacy=on \ + -chardev "file,id=con0,path=$WORK/$1.stdout,input-path=$WORK/$1.stdin" \ + -device virtconsole,chardev=con0,bus=vs0.0 +} + +# Run check: TA output, and the echoed input, on the console. +check_console() { + local out="$WORK/$1.stdout" + if [[ ${verbose:-0} -ne 0 ]]; then + echo "--- console ($1) ---" >&2 + cat "$out" >&2 + echo "---" >&2 + fi + grep -q "/TA:" "$out" || { echo "no TA output on the console"; return 1; } + if [[ $profile == debug ]] && ! grep -qF "[stdin] $STDIN_LINE" "$out"; then + echo "standard input was not echoed on the console" + return 1 + fi +} +check=check_console + for ta in "${tas[@]}"; do name=${ta%@default} if [[ -f "$VM_TESTS_DIR/$name-cmds.json" ]]; then @@ -118,6 +150,7 @@ for ta in "${tas[@]}"; do files+=(cmds.json) fi tar --format=ustar -cf "$WORK/$ta.tar" -C "$WORK/$ta" "${files[@]}" - run_vm "$ta" "$KERNEL" "$WORK/$ta.tar" + # shellcheck disable=SC2046 # console_args prints separate arguments + run_vm "$ta" "$KERNEL" "$WORK/$ta.tar" $(console_args "$ta") done summary diff --git a/litebox_bootloader/src/lib.rs b/litebox_bootloader/src/lib.rs index 27faf0350e..917b3e2a21 100644 --- a/litebox_bootloader/src/lib.rs +++ b/litebox_bootloader/src/lib.rs @@ -33,7 +33,8 @@ use layout::{ heap_start_address, rodata_end_address, rodata_start_address, text_end_address, text_start_address, }; -use litebox_platform_vm_kernel::{BootConfig, KERNEL_OFFSET, VmKernel, clock::TscClock}; +use litebox_hal::clock::Clock; +use litebox_platform_vm_kernel::{BootConfig, KERNEL_OFFSET, VmKernel}; use x86_64::PhysAddr; const PAGE_SIZE: u64 = 4096; @@ -43,7 +44,7 @@ pub struct Kernel { /// Boot modules stay mapped at `PA + KERNEL_OFFSET`, in kernel-managed /// RAM, for the kernel's lifetime. pub boot_info: &'static BootInfo, - pub tsc_khz: u64, + pub clock: Clock, } /// Declares the kernel's `fn(Kernel) -> !`, run on the kernel stack once the @@ -70,15 +71,15 @@ unsafe extern "Rust" { safe fn __litebox_kernel_logger() -> &'static dyn log::Log; } -static BOOT: spin::Once<(BootInfo, u64)> = spin::Once::new(); +static BOOT: spin::Once<(BootInfo, Clock)> = spin::Once::new(); fn early_console_init() { let _ = log::set_logger(__litebox_kernel_logger()); log::set_max_level(log::LevelFilter::Info); - litebox_hal::console_println!("====================="); - litebox_hal::console_println!(" Hello from LiteBox! "); - litebox_hal::console_println!("====================="); + litebox_hal::serial_println!("====================="); + litebox_hal::serial_println!(" Hello from LiteBox! "); + litebox_hal::serial_println!("====================="); } /// The front ends' entry; see [`handoff`]. @@ -101,10 +102,9 @@ fn kernel_start(boot_info: impl FnOnce() -> BootInfo) -> ! { .collect(); let to_pa = |va: u64| VmKernel::va_to_pa(x86_64::VirtAddr::new(va)); litebox_hal::interrupt::init_legacy_pics(); - let tsc_khz = litebox_hal::clock::calibrate_tsc_khz(); - litebox_util_log::info!(mhz:% = tsc_khz / 1000; "TSC calibrated against the PIT"); - BOOT.call_once(|| (info, tsc_khz)); - let clock = alloc::boxed::Box::leak(alloc::boxed::Box::new(TscClock::new(tsc_khz))); + let clock = Clock::measure(); + BOOT.call_once(|| (info, clock)); + let clock = alloc::boxed::Box::leak(alloc::boxed::Box::new(clock)); // Safety: the front end established the direct mapping with IRQs off on one // CPU. The heap contains only `ram`, text bounds come from the linker, and // no live resource needs the boot stack after the handoff. @@ -124,7 +124,7 @@ fn kernel_start(boot_info: impl FnOnce() -> BootInfo) -> ! { } fn kernel_main(platform: &'static VmKernel) -> ! { - let (boot_info, tsc_khz) = BOOT.get().expect("boot info captured"); + let (boot_info, clock) = BOOT.get().expect("boot info captured"); for m in &boot_info.modules { assert!( platform.contains_ram(m.clone()), @@ -136,7 +136,7 @@ fn kernel_main(platform: &'static VmKernel) -> ! { __litebox_kernel_main(Kernel { platform, boot_info, - tsc_khz: *tsc_khz, + clock: *clock, }) } diff --git a/litebox_broker_host/src/readiness.rs b/litebox_broker_host/src/readiness.rs index c928dabddf..ad2736b293 100644 --- a/litebox_broker_host/src/readiness.rs +++ b/litebox_broker_host/src/readiness.rs @@ -380,6 +380,24 @@ pub fn publish_readiness( } } +/// Publishes every pending readiness update, then returns; for deployments +/// without a publisher thread, which call this whenever updates may be +/// pending. +/// +/// The channel must not block: a send that cannot proceed (e.g., a full +/// notification ring) fails, which leaves its update publishable for the next +/// call and returns the error. +pub fn publish_pending_readiness( + publisher: &ReadinessPublisher, + channel: &mut Channel, +) -> Result<(), Channel::Error> { + while let Some(pending) = publisher.take_pending() { + channel.send_notification(&BrokerNotification::Readiness(pending.notification()))?; + pending.confirm(); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -402,6 +420,28 @@ mod tests { drained } + #[test] + fn pending_readiness_survives_a_failed_nonblocking_send() { + struct Full(bool); + impl HostNotificationChannel for Full { + type Error = (); + fn send_notification(&mut self, _: &BrokerNotification) -> Result<(), ()> { + if self.0 { Err(()) } else { Ok(()) } + } + } + let publisher = ReadinessPublisher::new(); + publish(&publisher, HANDLE, ReadinessFlags::READ); + assert_eq!( + publish_pending_readiness(&publisher, &mut Full(true)), + Err(()) + ); + assert_eq!( + publish_pending_readiness(&publisher, &mut Full(false)), + Ok(()) + ); + assert!(publisher.take_pending().is_none()); + } + #[test] fn repeating_known_readiness_publishes_nothing() { let publisher = ReadinessPublisher::new(); diff --git a/litebox_broker_local_vm_userland/src/lib.rs b/litebox_broker_local_vm_userland/src/lib.rs index 8e6046a869..69641f99f8 100644 --- a/litebox_broker_local_vm_userland/src/lib.rs +++ b/litebox_broker_local_vm_userland/src/lib.rs @@ -10,6 +10,10 @@ //! (`BrokerEnter`'s Wake and Wait). The kernel serves the ring within each, //! so a call takes one entry. Shared memory is accessed only through //! `peer_memory`. +//! +//! Notifications (readiness of broker objects, e.g., standard input) have no +//! receiver thread: the runner receives them with [`Notifications::receive`], +//! which also waits for them. #![cfg(target_arch = "x86_64")] #![no_std] @@ -21,7 +25,8 @@ use alloc::sync::Arc; use litebox::utils::TruncateExt as _; use litebox_broker_local::BrokerLocal; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerRequest, BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, + BrokerResponse, }; use litebox_broker_protocol::wire::{self, WireError}; use litebox_broker_transport::channel::{LocalCallChannel, LocalSetupChannel}; @@ -98,8 +103,7 @@ impl LocalSetupChannel for KernelBrokerSetup { } } -/// The local ends of the request and response rings. The notification -/// consumer awaits the kernel's first notifying provider (e.g., timers). +/// The local ends of the request and response rings. struct Rings { requests: ControlRingProducer, responses: ControlRingConsumer, @@ -334,6 +338,13 @@ impl WaitableSharedMemory for KernelControlRing { } } +/// The notification ring's local end. +pub struct Notifications { + consumer: ControlRingConsumer, + /// For [`self_check`]. + words: KernelControlRing, +} + /// Call at most once per process. /// /// # Errors @@ -341,30 +352,81 @@ impl WaitableSharedMemory for KernelControlRing { /// The kernel rejects the association or negotiation fails. pub fn connect( info: &StartupInfo, -) -> litebox_broker_local::Result, ChannelError> { - let (local, _startup, ()) = BrokerLocal::negotiate(KernelBrokerSetup::default(), |_setup| { - let memory = KernelControlRing::new(info.broker_control_ring) - .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange))?; - let LocalControlRingEndpoints { - request_producer, - response_consumer, - // Unused until the kernel produces notifications. - notification_consumer: _, - } = ControlRing::new(memory) - .map_err(ChannelError::Ring)? - .into_local(); - let channel = KernelBrokerChannel { - rings: spin::Mutex::new(Some(Rings { - requests: request_producer, - responses: response_consumer, - })), - }; - let shared_memory: Arc = Arc::new(KernelSharedMemory { - region: info.broker_shared_memory, - }); - Ok((channel, shared_memory, ())) - })?; - Ok(local) +) -> litebox_broker_local::Result<(BrokerLocal, Notifications), ChannelError> { + let (local, _startup, notifications) = + BrokerLocal::negotiate(KernelBrokerSetup::default(), |_setup| { + let ring = || { + KernelControlRing::new(info.broker_control_ring) + .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange)) + }; + let LocalControlRingEndpoints { + request_producer, + response_consumer, + notification_consumer, + } = ControlRing::new(ring()?) + .map_err(ChannelError::Ring)? + .into_local(); + let notifications = Notifications { + consumer: notification_consumer, + words: ring()?, + }; + let channel = KernelBrokerChannel { + rings: spin::Mutex::new(Some(Rings { + requests: request_producer, + responses: response_consumer, + })), + }; + let shared_memory: Arc = Arc::new(KernelSharedMemory { + region: info.broker_shared_memory, + }); + Ok((channel, shared_memory, notifications)) + })?; + Ok((local, notifications)) +} + +impl Notifications { + /// Passes every published notification to `dispatch` (e.g., + /// `LiteBox::broker_notification_dispatcher`); if there is none, first + /// waits for one until `deadline` (TSC; zero for none). Returns how many + /// it passed: zero at the deadline. + /// + /// # Errors + /// + /// A malformed ring, or a wait the kernel rejects ([`Status::Stalled`] + /// if nothing could publish). + pub fn receive( + &mut self, + deadline: u64, + mut dispatch: impl FnMut(BrokerNotification), + ) -> Result { + let mut received = 0; + loop { + match self.consumer.try_read(wire::decode_notification) { + Ok(ControlRingReadStatus::Message(notification)) => { + dispatch(notification); + received += 1; + } + Ok(ControlRingReadStatus::Empty { wait_epoch }) => { + if received != 0 { + break; + } + let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + match broker_wait(epoch, wait_epoch, deadline) { + Ok(()) => {} + Err(Status::TimedOut) => return Ok(0), + Err(status) => return Err(ChannelError::Kernel(status)), + } + } + Err(ControlRingReadError::Ring(error)) => return Err(ChannelError::Ring(error)), + Err(ControlRingReadError::Decode(error)) => { + return Err(ChannelError::Wire(error)); + } + } + } + // The kernel sees the freed slots when it next publishes. + self.consumer.publish_head().map_err(ChannelError::Ring)?; + Ok(received) + } } /// Serves published broker requests. @@ -379,17 +441,40 @@ fn broker_wait(offset: usize, expected: u32, deadline: u64) -> Result<(), Status } /// Self-check (debug builds): the broker's futex operations, on the -/// notification ring's producer epoch, which stays zero (no notifications). +/// notification ring's producer epoch and on the request ring's consumer +/// epoch, which only the kernel's serving changes. /// /// # Panics /// /// On a failed check. -pub fn self_check() { +pub fn self_check(notifications: &Notifications, tsc_khz: u64) { let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + let served = ControlRingDirection::Requests.consumer_epoch_offset(); + // Safety: RDTSC has no side effects. + let now = || unsafe { core::arch::x86_64::_rdtsc() }; + let current = |offset| { + notifications + .words + .load_u32_acquire(offset) + .expect("a ring word") + }; broker_wake().expect("waking an idle broker"); - assert_eq!(broker_wait(epoch, 1, 0), Ok(()), "word differs"); - assert_eq!(broker_wait(epoch, 0, 0), Err(Status::Stalled)); - assert_eq!(broker_wait(epoch, 0, 1), Err(Status::Unsupported)); + let notified = current(epoch); + assert_eq!( + broker_wait(epoch, notified.wrapping_add(1), 0), + Ok(()), + "word differs" + ); + let soon = now() + tsc_khz; // 1 ms + assert_eq!( + broker_wait(served, current(served), 0), + Err(Status::Stalled) + ); + assert_eq!( + broker_wait(served, current(served), soon), + Err(Status::TimedOut) + ); + assert!(now() >= soon, "a timed wait returned early"); assert_eq!(broker_wait(0, 0, 0), Err(Status::InvalidArgument)); let mut wake = BrokerEnterRequest::wake(); wake.expected = 1; diff --git a/litebox_broker_vm_kernel/Cargo.toml b/litebox_broker_vm_kernel/Cargo.toml index 859e7c1bd0..1b9985c12b 100644 --- a/litebox_broker_vm_kernel/Cargo.toml +++ b/litebox_broker_vm_kernel/Cargo.toml @@ -14,6 +14,7 @@ litebox_common_vm_abi = { path = "../litebox_common_vm_abi", version = "0.1.0" } litebox_hal = { path = "../litebox_hal", version = "0.1.0" } litebox_platform_vm_kernel = { path = "../litebox_platform_vm_kernel", version = "0.1.0" } log = { version = "0.4", default-features = false } +spin = { version = "0.10.0", default-features = false, features = ["spin_mutex"] } [lints] workspace = true diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index a1ec7cd606..b49579ec25 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -7,6 +7,11 @@ //! responses are in the control ring, pinned and accessed through the //! kernel's mapping; payloads are in the process's lazily populated shared //! buffers. +//! +//! Files are the devices under `/dev`, with standard streams on the +//! runner's [`stdio::Console`]. Readiness (e.g., of standard input) goes to the +//! association's notification ring. Waiting ([`BrokerEnterOp::Wait`]) halts +//! until the runner's [`Events`] report something, e.g., an interrupt. #![cfg(target_arch = "x86_64")] #![no_std] @@ -16,6 +21,8 @@ extern crate alloc; mod memory; mod providers; +mod readiness; +pub mod stdio; use alloc::boxed::Box; use alloc::sync::Arc; @@ -25,21 +32,22 @@ use core::convert::Infallible; use litebox_broker_core::BrokerCore; use litebox_broker_core::readiness::ReadinessSink; use litebox_broker_host::BrokerHostAssociation; -use litebox_broker_protocol::ObjectHandle; use litebox_broker_protocol::message::{BrokerHandshakeRequest, BrokerHandshakeResponse}; -use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_protocol::wire; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::control_ring::{ BrokerControlRingEndpoints, CONTROL_RING_MEMORY_SIZE, ControlRing, ControlRingConsumer, - ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, MemoryAccessPolicy, + ControlRingDirection, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, + MemoryAccessPolicy, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool}; use litebox_common_vm_abi::{BrokerEnterOp, BrokerEnterRequest, Status, UserRange, WireFrame}; use litebox_platform_vm_kernel::PinnedUserPages; use memory::{PinnedControlRing, UserSharedMemory}; +use readiness::KernelReadiness; +use stdio::ConsoleStdio; /// Size of a process's broker shared memory. pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; @@ -47,10 +55,29 @@ pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; /// Size of a process's broker control ring. pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; +/// What the broker waits for: the runner's event sources (e.g., device +/// interrupts and a timer). +pub trait Events: Sync { + /// Halts until an event may have happened, or until `deadline` (a TSC + /// value, from ring 3; see [`litebox_hal::clock::now`]) passes. Without a + /// deadline, false if no event could ever end the halt (and it does not + /// halt). + fn wait(&self, deadline: Option) -> bool; +} + +/// The runner's sources for a [`Broker`]. +pub struct Config { + /// Behind `/dev/stdin`, `/dev/stdout`, and `/dev/stderr`. + pub console: &'static dyn stdio::Console, + pub events: &'static dyn Events, +} + /// The kernel's broker; clones share it. #[derive(Clone)] pub struct Broker { core: BrokerCore, + events: &'static dyn Events, + stdio: Arc, } impl Broker { @@ -59,13 +86,12 @@ impl Broker { /// # Panics /// /// Without a hardware CSPRNG, or on a second call. - #[expect( - clippy::new_without_default, - reason = "not `Default`: a second call panics" - )] - pub fn new() -> Self { + pub fn new(config: Config) -> Self { + let stdio = Arc::new(ConsoleStdio::new(config.console)); Self { - core: providers::core(), + core: providers::core(stdio.clone()), + events: config.events, + stdio, } } @@ -87,40 +113,12 @@ impl Broker { Association { broker: self.clone(), shared_memory, + readiness: Arc::new(KernelReadiness::default()), state: RefCell::new(State::New(ring)), } } } -/// Discards readiness: the broker provides only randomness so far. A provider -/// with readiness (e.g., timers) needs a sink that publishes to the -/// notification ring. -struct NoReadiness; - -impl ReadinessSink for NoReadiness { - fn max_tracked_objects(&self) -> usize { - usize::MAX - } - - fn publish( - &self, - _handle: ObjectHandle, - _readiness: ReadinessFlags, - ) -> litebox_broker_core::Result<()> { - Ok(()) - } - - fn republish( - &self, - _handle: ObjectHandle, - _readiness: ReadinessFlags, - ) -> litebox_broker_core::Result<()> { - Ok(()) - } - - fn retire(&self, _handle: ObjectHandle) {} -} - /// One handshake request in, one response out. struct KernelHostSetup { request: Option, @@ -161,8 +159,8 @@ enum State { Ended, } -/// The broker's ends of the request and response rings. Its notification -/// producer awaits a provider with readiness (see `NoReadiness`). +/// The broker's ends of the request and response rings; the notification +/// ring's is in [`Association::readiness`]. struct Active { association: BrokerHostAssociation, requests: ControlRingConsumer, @@ -179,6 +177,7 @@ struct Active { pub struct Association { broker: Broker, shared_memory: UserRange, + readiness: Arc, state: RefCell, } @@ -211,7 +210,7 @@ impl Association { request: Some(request), response: None, }; - let readiness: Arc = Arc::new(NoReadiness); + let readiness: Arc = self.readiness.clone(); let result = litebox_broker_host::setup_connection( &self.broker.core, None, @@ -232,9 +231,9 @@ impl Association { let BrokerControlRingEndpoints { request_consumer, response_producer, - // Unused until a provider has readiness. - notification_producer: _, + notification_producer, } = ring.into_broker(); + self.readiness.activate(notification_producer); *state = State::Active(Box::new(Active { association: host, requests: request_consumer, @@ -252,6 +251,13 @@ impl Association { /// performs `request`'s operation. An invalid request has no effect. /// Before the handshake, any request is [`Status::Denied`]. /// + /// Waiting halts the kernel (single CPU, no scheduler) until the word + /// changes or the deadline passes. Only notifications change ring words + /// asynchronously; a wait on another word without a deadline, or on the + /// notification epoch when no event could publish, is + /// [`Status::Stalled`]. A corrupted notification ring fails the + /// association, as a corrupted request ring does. + /// /// # Errors /// /// See [`EnterError`]. @@ -260,6 +266,20 @@ impl Association { let State::Active(active) = &mut *state else { return Err(EnterError::Status(Status::Denied)); }; + let result = self.enter_active(active, request); + if result == Err(EnterError::Failed) + && let State::Active(active) = core::mem::replace(&mut *state, State::Ended) + { + end(active.association, &self.readiness); + } + result + } + + fn enter_active( + &self, + active: &mut Active, + request: &BrokerEnterRequest, + ) -> Result<(), EnterError> { let invalid = || EnterError::Status(Status::InvalidArgument); let wait_offset = match request.op { BrokerEnterOp::Wake => { @@ -269,9 +289,6 @@ impl Association { None } BrokerEnterOp::Wait => { - if request.deadline != 0 { - return Err(EnterError::Status(Status::Unsupported)); - } let offset = usize::try_from(request.offset).map_err(|_| invalid())?; if !MemoryAccessPolicy::ControlRing.permits_u32(offset) { return Err(invalid()); @@ -279,20 +296,42 @@ impl Association { Some(offset) } }; - if active.serve().is_err() { - if let State::Active(active) = core::mem::replace(&mut *state, State::Ended) { - end(active.association); + self.broker.stdio.poll(); + active.serve().map_err(|Failed| EnterError::Failed)?; + self.flush_readiness()?; + let Some(offset) = wait_offset else { + return Ok(()); + }; + let deadline = (request.deadline != 0).then_some(request.deadline); + let asynchronous = offset == ControlRingDirection::Notifications.producer_epoch_offset(); + loop { + match active.words.load_u32_acquire(offset) { + Ok(word) if word != request.expected => return Ok(()), + Ok(_) => {} + Err(_) => return Err(invalid()), } - return Err(EnterError::Failed); + if deadline.is_some_and(|deadline| litebox_hal::clock::now() >= deadline) { + return Err(EnterError::Status(Status::TimedOut)); + } + if deadline.is_none() && !asynchronous { + return Err(EnterError::Status(Status::Stalled)); + } + if !self.broker.events.wait(deadline) { + return Err(EnterError::Status(Status::Stalled)); + } + self.broker.stdio.poll(); + self.flush_readiness()?; } - // Nothing else changes ring words, so a word unchanged after serving - // stays so. - match wait_offset.map(|offset| active.words.load_u32_acquire(offset)) { - None => Ok(()), - Some(Ok(word)) if word != request.expected => Ok(()), - Some(Ok(_)) => Err(EnterError::Status(Status::Stalled)), - Some(Err(_)) => Err(invalid()), + } + + /// Publishes pending readiness; a corrupted notification ring fails the + /// association. + fn flush_readiness(&self) -> Result<(), EnterError> { + self.readiness.flush(); + if self.readiness.failed() { + return Err(EnterError::Failed); } + Ok(()) } } @@ -369,7 +408,8 @@ pub enum EnterError { /// Ends an active association as the userland broker does when its runner /// goes away. -fn end(association: BrokerHostAssociation) { +fn end(association: BrokerHostAssociation, readiness: &KernelReadiness) { + readiness.close(); association.request_cancellation(); association.association_ending(); association.finish(); @@ -378,7 +418,7 @@ fn end(association: BrokerHostAssociation) { impl Drop for Association { fn drop(&mut self) { if let State::Active(active) = core::mem::replace(self.state.get_mut(), State::Ended) { - end(active.association); + end(active.association, &self.readiness); } } } diff --git a/litebox_broker_vm_kernel/src/providers.rs b/litebox_broker_vm_kernel/src/providers.rs index 9363dab670..6960c18d63 100644 --- a/litebox_broker_vm_kernel/src/providers.rs +++ b/litebox_broker_vm_kernel/src/providers.rs @@ -1,16 +1,19 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! The broker core's providers: hardware randomness only. +//! The broker core's providers: hardware randomness, and `/dev` (standard +//! streams from the runner's [`StdioProvider`], `null`, `urandom`). use alloc::sync::Arc; use litebox_broker_core::{ BrokerCore, ObjectRights, PolicyEngine, - fs::UnsupportedFileService, + fs::{FileService, composer::Composer, devices::Devices, resolver::Resolver}, random::{RandomProvider, RandomProviderError}, socket::UnsupportedSocketProvider, + stdio::StdioProvider, timer::UnsupportedTimerProvider, }; +use litebox_platform_vm_kernel::VmKernel; /// The HAL's hardware CSPRNG as the broker's randomness. struct HardwareRandom(litebox_hal::rng::Hardware); @@ -21,16 +24,31 @@ impl RandomProvider for HardwareRandom { } } +fn file_service( + stdio: Arc, + random: Arc, +) -> Arc { + let devices = Composer::builder() + .mount("/dev", |allocator| Devices::new(allocator, stdio, random)) + .build() + .unwrap_or_else(|error| panic!("broker file service: {error:?}")); + Arc::new(Resolver::::new(devices)) +} + +/// Runner processes, which the kernel vouches for, may open and use files; +/// the only ones are the devices. +/// /// # Panics /// /// Without a hardware CSPRNG, or on a second call. -pub(crate) fn core() -> BrokerCore { +pub(crate) fn core(stdio: Arc) -> BrokerCore { + let random: Arc = Arc::new(HardwareRandom(litebox_hal::rng::hardware())); BrokerCore::new( - PolicyEngine::with_host_guaranteed_rights(ObjectRights::empty()), + PolicyEngine::with_host_guaranteed_rights(ObjectRights::WAIT | ObjectRights::WRITE), Arc::new(UnsupportedSocketProvider), - Arc::new(HardwareRandom(litebox_hal::rng::hardware())), + Arc::clone(&random), Arc::new(UnsupportedTimerProvider), - Arc::new(UnsupportedFileService), + file_service(stdio, random), ) .unwrap_or_else(|error| panic!("broker core: {error:?}")) } diff --git a/litebox_broker_vm_kernel/src/readiness.rs b/litebox_broker_vm_kernel/src/readiness.rs new file mode 100644 index 0000000000..6646e7acca --- /dev/null +++ b/litebox_broker_vm_kernel/src/readiness.rs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! An association's readiness, published to its (pinned) notification ring +//! at once, whichever process is current. What a full ring cannot take +//! stays pending until the next [`KernelReadiness::flush`]. + +use core::sync::atomic::{AtomicBool, Ordering}; +use litebox_broker_core::readiness::ReadinessSink; +use litebox_broker_core::{BrokerError, Result}; +use litebox_broker_host::readiness::{ + MAX_TRACKED_READINESS_OBJECTS, ReadinessPublisher, publish_pending_readiness, +}; +use litebox_broker_protocol::ObjectHandle; +use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::readiness::ReadinessFlags; +use litebox_broker_protocol::wire; +use litebox_broker_transport::channel::HostNotificationChannel; +use litebox_broker_transport::control_ring::{ + ControlRingError, ControlRingProducer, ControlRingWriteStatus, +}; +use spin::mutex::SpinMutex; + +use crate::memory::PinnedControlRing; + +type Producer = ControlRingProducer; + +#[derive(Default)] +pub(crate) struct KernelReadiness { + publisher: ReadinessPublisher, + /// Until the handshake and after the association ends: `None`. + notifications: SpinMutex>, + /// The process corrupted its notification ring. + failed: AtomicBool, +} + +impl KernelReadiness { + /// Starts publishing to the ring, including what is already pending. + pub(crate) fn activate(&self, notifications: Producer) { + *self.notifications.lock() = Some(notifications); + self.flush(); + } + + /// Stops publishing, for good. + pub(crate) fn close(&self) { + self.publisher.close(); + *self.notifications.lock() = None; + } + + /// Publishes what is pending, as far as the ring has room. + pub(crate) fn flush(&self) { + // Nothing publishes with this lock held: sending only writes the + // ring, and interrupt handlers do not run sources. + let mut notifications = self.notifications.lock(); + let Some(producer) = notifications.as_mut() else { + return; + }; + match publish_pending_readiness(&self.publisher, &mut Ring(producer)) { + Ok(()) | Err(RingError::Full) => {} + Err(RingError::Ring(error)) => { + log::warn!("broker notification ring failed: {error:?}"); + *notifications = None; + self.publisher.close(); + self.failed.store(true, Ordering::Relaxed); + } + } + } + + /// Whether the process corrupted its notification ring, which fails the + /// association. + pub(crate) fn failed(&self) -> bool { + self.failed.load(Ordering::Relaxed) + } +} + +enum RingError { + Full, + Ring(ControlRingError), +} + +/// Never blocks: a full ring fails the send. +struct Ring<'a>(&'a mut Producer); + +impl HostNotificationChannel for Ring<'_> { + type Error = RingError; + + fn send_notification( + &mut self, + notification: &BrokerNotification, + ) -> core::result::Result<(), RingError> { + let frame = wire::encode_notification(notification.clone()); + match self.0.try_write(&frame).map_err(RingError::Ring)? { + ControlRingWriteStatus::Written => Ok(()), + ControlRingWriteStatus::Full { .. } => Err(RingError::Full), + } + } +} + +impl ReadinessSink for KernelReadiness { + fn max_tracked_objects(&self) -> usize { + MAX_TRACKED_READINESS_OBJECTS + } + + fn publish(&self, handle: ObjectHandle, readiness: ReadinessFlags) -> Result<()> { + self.publisher + .publish(handle, readiness) + .map_err(|_| BrokerError::ResourceExhausted)?; + self.flush(); + Ok(()) + } + + fn republish(&self, handle: ObjectHandle, readiness: ReadinessFlags) -> Result<()> { + self.publisher + .republish(handle, readiness) + .map_err(|_| BrokerError::ResourceExhausted)?; + self.flush(); + Ok(()) + } + + fn retire(&self, handle: ObjectHandle) { + self.publisher.retire(handle); + } +} diff --git a/litebox_broker_vm_kernel/src/stdio.rs b/litebox_broker_vm_kernel/src/stdio.rs new file mode 100644 index 0000000000..7290d721e0 --- /dev/null +++ b/litebox_broker_vm_kernel/src/stdio.rs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Runner processes' standard streams, on the runner's [`Console`]; both +//! output streams share it. Standard-input files are woken when input starts +//! to be pending. + +use core::sync::atomic::{AtomicBool, Ordering}; +use litebox_broker_core::readiness::{ReadinessRegistration, ReadinessWatchers}; +use litebox_broker_core::stdio::{ + StdioOutputStream, StdioProvider, StdioProviderError, StdioStream, +}; +use litebox_broker_protocol::readiness::ReadinessFlags; +use spin::Mutex; + +/// A byte-stream device behind the standard streams. +pub trait Console: Sync { + /// Copies received bytes into `output`: how many (zero if none yet), or + /// `None` at the end of input (e.g., a device without input). + fn read(&self, output: &mut [u8]) -> Option; + + /// Whether [`Self::read`] would not return zero: input or its end. + fn has_input(&self) -> bool; + + /// How much of `input` the device took; less if it stays full. + fn write(&self, input: &[u8]) -> usize; +} + +pub(crate) struct ConsoleStdio { + console: &'static dyn Console, + /// Standard-input files, to wake when input arrives. + watchers: Mutex, + /// Whether input was pending when last looked at. + readable: AtomicBool, +} + +impl ConsoleStdio { + pub(crate) fn new(console: &'static dyn Console) -> Self { + Self { + console, + watchers: Mutex::new(ReadinessWatchers::default()), + readable: AtomicBool::new(false), + } + } + + /// Wakes standard-input watchers if input started to be pending. + pub(crate) fn poll(&self) { + let readable = self.console.has_input(); + if readable && !self.readable.swap(true, Ordering::Relaxed) { + self.watchers.lock().publish(ReadinessFlags::READ); + } else if !readable { + self.readable.store(false, Ordering::Relaxed); + } + } +} + +impl StdioProvider for ConsoleStdio { + fn read(&self, output: &mut [u8]) -> Result { + match self.console.read(output) { + None => Ok(0), + Some(0) => Err(StdioProviderError::WouldBlock), + Some(read) => Ok(read), + } + } + + fn write(&self, _stream: StdioOutputStream, input: &[u8]) -> Result { + match self.console.write(input) { + 0 => Err(StdioProviderError::WouldBlock), + written => Ok(written), + } + } + + /// No terminal control. + fn is_terminal(&self, _stream: StdioStream) -> bool { + false + } + + fn readiness(&self, stream: StdioStream) -> ReadinessFlags { + match stream { + StdioStream::Stdin if !self.console.has_input() => ReadinessFlags(0), + StdioStream::Stdin => ReadinessFlags::READ, + StdioStream::Stdout | StdioStream::Stderr => ReadinessFlags::WRITE, + } + } + + fn watch( + &self, + stream: StdioStream, + registration: &ReadinessRegistration, + ) -> litebox_broker_core::Result<()> { + if stream == StdioStream::Stdin { + self.watchers.lock().watch(registration)?; + if self.console.has_input() { + registration.publish(ReadinessFlags::READ)?; + } + } + Ok(()) + } +} diff --git a/litebox_common_vm_abi/src/lib.rs b/litebox_common_vm_abi/src/lib.rs index f05077be5e..57b15c7c43 100644 --- a/litebox_common_vm_abi/src/lib.rs +++ b/litebox_common_vm_abi/src/lib.rs @@ -91,8 +91,9 @@ //! [`BrokerEnterOp::Wake`] and [`BrokerEnterOp::Wait`] on ring words. The //! kernel serves every published request first. //! - The ring's notification direction carries readiness of broker objects -//! (e.g., timers) from the kernel. The kernel's broker provides only -//! randomness so far, so it produces none yet. +//! (e.g., standard input) from the kernel, published as it changes; the +//! runner consumes them and waits for them on the direction's producer +//! epoch. //! - The kernel pins the control ring and accesses it through its own mapping. //! The shared buffers stay lazily populated and are touched only while //! executing a request that names them. @@ -591,8 +592,9 @@ pub enum BrokerEnterOp { Wake = 0, /// Returns once the `u32` ring word at `offset`, an epoch, differs from /// `expected`. [`Status::TimedOut`] at the deadline; [`Status::Stalled`] - /// if nothing could change the word. Deadlines are - /// [`Status::Unsupported`] so far. + /// if nothing could change the word. Only notifications change words + /// while the kernel waits (with no scheduler, it halts the CPU), so + /// without a deadline any other word stalls. Wait = 1, } diff --git a/litebox_hal/src/clock.rs b/litebox_hal/src/clock.rs index d80877bee0..ed86e23095 100644 --- a/litebox_hal/src/clock.rs +++ b/litebox_hal/src/clock.rs @@ -1,18 +1,116 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Clocks: TSC frequency, calibrated against the legacy PIT. +//! The CPU's clock: a free-running counter ([`now`], in ticks) and its +//! frequency ([`Clock`]). On x86-64, the counter is the TSC, which ring 3 +//! reads too: `BrokerEnter` deadlines are TSC values and +//! `StartupInfo::tsc_khz` is [`Clock::khz`] (see `litebox_common_vm_abi`), so +//! the TSC is part of the ABI. Its frequency comes from the most +//! authoritative source the machine has: CPUID, the hypervisor, or else +//! calibration against the legacy PIT (which, e.g., Firecracker and Cloud +//! Hypervisor lack). + +use core::arch::x86_64::__cpuid_count as cpuid; +use core::time::Duration; const PIT_HZ: u64 = 1_193_182; const CALIBRATION_MS: u64 = 50; const CALIBRATION_TIMEOUT_CYCLES: u64 = 1 << 36; +/// Frequencies outside this range are not believed. +const PLAUSIBLE_KHZ: core::ops::RangeInclusive = 100_000..=100_000_000; + +/// The counter: the TSC, as ring 3 reads it (see the module docs). Never +/// decreases (one CPU, constant rate). +pub fn now() -> u64 { + // Safety: RDTSC has no side effects. + unsafe { core::arch::x86_64::_rdtsc() } +} + +/// The counter's frequency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Clock { + khz: u64, +} + +impl Clock { + /// Finds the frequency (logged, with its source). Call once, at boot; + /// calibration takes a while. + /// + /// # Panics + /// + /// If no source works. + pub fn measure() -> Self { + let (khz, source) = tsc_khz(); + litebox_util_log::info!(mhz:% = khz / 1000, source:? = source; "clock frequency"); + Self { khz } + } + + /// Ticks per millisecond: the TSC frequency in kHz, as ring 3 gets it. + pub fn khz(self) -> u64 { + self.khz + } + + /// [`now`] in nanoseconds. + pub fn nanos(self) -> u64 { + u64::try_from(u128::from(now()) * 1_000_000 / u128::from(self.khz)).unwrap_or(u64::MAX) + } + + /// The tick [`now`] reaches `timeout` from now. + pub fn deadline(self, timeout: Duration) -> u64 { + let ticks = timeout.as_nanos() * u128::from(self.khz) / 1_000_000; + now().saturating_add(u64::try_from(ticks).unwrap_or(u64::MAX)) + } +} + +/// Where [`tsc_khz`] got the frequency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TscSource { + /// CPUID leaf 0x15: the core crystal and its ratio (Intel). + Cpuid, + /// The hypervisor's timing leaf 0x40000010 (KVM via QEMU, VMware). + Hypervisor, + /// Calibration against the PIT. + Pit, +} + +/// The TSC frequency in kHz, and its source. +fn tsc_khz() -> (u64, TscSource) { + if let Some(khz) = cpuid_khz().filter(|khz| PLAUSIBLE_KHZ.contains(khz)) { + return (khz, TscSource::Cpuid); + } + if let Some(khz) = hypervisor_khz().filter(|khz| PLAUSIBLE_KHZ.contains(khz)) { + return (khz, TscSource::Hypervisor); + } + (calibrate_tsc_khz(), TscSource::Pit) +} + +fn cpuid_khz() -> Option { + if cpuid(0, 0).eax < 0x15 { + return None; + } + let leaf = cpuid(0x15, 0); + let (denominator, numerator, crystal_hz) = (leaf.eax, leaf.ebx, leaf.ecx); + (denominator != 0 && numerator != 0 && crystal_hz != 0) + .then(|| u64::from(crystal_hz) * u64::from(numerator) / u64::from(denominator) / 1000) +} + +fn hypervisor_khz() -> Option { + const KVM: [u32; 3] = [0x4b4d_564b, 0x564b_4d56, 0x0000_004d]; // "KVMKVMKVM\0\0\0" + const VMWARE: [u32; 3] = [0x6177_4d56, 0x4d56_6572, 0x6572_6177]; // "VMwareVMware" + if cpuid(1, 0).ecx & (1 << 31) == 0 { + return None; + } + let leaf = cpuid(0x4000_0000, 0); + let signature = [leaf.ebx, leaf.ecx, leaf.edx]; + if signature != KVM && signature != VMWARE || leaf.eax < 0x4000_0010 { + return None; + } + Some(u64::from(cpuid(0x4000_0010, 0).eax)) +} + /// Uses PIT channel 2 exclusively for ~50 ms. -/// -/// # Panics -/// -/// Panics if calibration fails. -pub fn calibrate_tsc_khz() -> u64 { +fn calibrate_tsc_khz() -> u64 { use x86_64::instructions::port::Port; const PORT_B: u16 = 0x61; // gate: bit 0, speaker: bit 1, OUT2: bit 5 @@ -32,18 +130,18 @@ pub fn calibrate_tsc_khz() -> u64 { ch2.write(hi); // Mode 0 starts counting on the PIT clock after the MSB write. - let start = core::arch::x86_64::_rdtsc(); - let mut now = start; + let start = now(); + let mut end = start; while port_b.read() & 0x20 == 0 { - now = core::arch::x86_64::_rdtsc(); + end = now(); assert!( - now - start < CALIBRATION_TIMEOUT_CYCLES, + end - start < CALIBRATION_TIMEOUT_CYCLES, "TSC calibration: the legacy PIT (channel 2) never signalled" ); } // Bits 4-7 are read-only. port_b.write(v & 0x0f); - let khz = (now - start) / CALIBRATION_MS; + let khz = (end - start) / CALIBRATION_MS; assert!(khz != 0, "TSC calibration measured a zero frequency"); khz } diff --git a/litebox_hal/src/devices.rs b/litebox_hal/src/devices.rs new file mode 100644 index 0000000000..6373641c93 --- /dev/null +++ b/litebox_hal/src/devices.rs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Devices behind device-neutral types ([`ConsoleDevice`]), each probing +//! the drivers the HAL has for its kind. Runners choose which devices to +//! probe, where they interrupt, and what to do without one. + +use crate::dma::Hal; +use crate::interrupt::InterruptTarget; +use crate::virtio::console::VirtioConsole; +use alloc::boxed::Box; +use core::fmt; + +/// A console: output, and input if the device receives any. Not internally +/// synchronized. +pub struct ConsoleDevice(Backend); + +enum Backend { + Virtio(Box), + /// The kernel's UART ([`crate::serial`]): output only. + Serial, +} + +impl ConsoleDevice { + /// The VM's console device, interrupting `interrupt` if it can (see + /// [`Self::interrupts`]); `None` if there is none. + /// + /// # Errors + /// + /// The device cannot be driven. + pub fn probe( + hal: &'static dyn Hal, + interrupt: InterruptTarget, + ) -> Result, crate::virtio::Error> { + // Virtio over PCI. + let device = VirtioConsole::probe(hal, interrupt.msi_message()).transpose()?; + Ok(device.map(|device| Self(Backend::Virtio(Box::new(device))))) + } + + /// Output to the kernel's UART; receives nothing. + pub fn serial() -> Self { + Self(Backend::Serial) + } + + /// Whether the device receives input at all. + pub fn receives(&self) -> bool { + match &self.0 { + Backend::Virtio(_) => true, + Backend::Serial => false, + } + } + + /// Whether the device raises its interrupt; else it must be polled. + pub fn interrupts(&self) -> bool { + match &self.0 { + Backend::Virtio(device) => device.transport().interrupts(), + Backend::Serial => false, + } + } + + /// Queues as much of `bytes` as the device takes now; returns how much. + pub fn write(&mut self, bytes: &[u8]) -> usize { + match &mut self.0 { + Backend::Virtio(device) => device.write(bytes), + Backend::Serial => { + crate::serial::print_bytes(bytes); + bytes.len() + } + } + } + + /// Whether received input is waiting. + pub fn has_input(&mut self) -> bool { + match &mut self.0 { + Backend::Virtio(device) => device.has_input(), + Backend::Serial => false, + } + } + + /// Copies received input into `output`; returns how much (zero if none + /// is waiting). + pub fn read(&mut self, output: &mut [u8]) -> usize { + match &mut self.0 { + Backend::Virtio(device) => device.read(output), + Backend::Serial => 0, + } + } +} + +impl fmt::Display for ConsoleDevice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.0 { + Backend::Virtio(device) => write!( + f, + "virtio (pci {}, interrupts {})", + device.transport().function(), + device.transport().interrupts() + ), + Backend::Serial => f.write_str("serial (output only)"), + } + } +} diff --git a/litebox_hal/src/dma.rs b/litebox_hal/src/dma.rs new file mode 100644 index 0000000000..d4bf2c72b0 --- /dev/null +++ b/litebox_hal/src/dma.rs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! What device drivers need from the kernel: memory a device can access +//! (DMA) and device registers (MMIO). The kernel platform implements +//! [`Hal`]. + +use core::ptr::NonNull; + +pub const PAGE_SIZE: usize = 4096; + +/// Zeroed, physically contiguous pages from a [`Hal`], which the kernel +/// accesses at `va` and devices at `pa`; returned to the [`Hal`] on drop, so +/// drop a region only once no device can access it (nothing in it is posted). +pub struct DmaRegion { + va: NonNull, + pa: u64, + len: usize, + hal: &'static dyn Hal, +} + +// Safety: a region is plain memory owned by whoever holds it. +unsafe impl Send for DmaRegion {} +// Safety: as above; shared access goes through raw pointers only. +unsafe impl Sync for DmaRegion {} + +impl DmaRegion { + /// At least `len` bytes; `None` when out of memory. + pub fn new(hal: &'static dyn Hal, len: usize) -> Option { + let pages = len.div_ceil(PAGE_SIZE).max(1); + let (va, pa) = hal.dma_alloc(pages)?; + let len = pages * PAGE_SIZE; + // Safety: freshly allocated and exclusively owned (`Hal` contract). + unsafe { va.write_bytes(0, len) }; + Some(Self { va, pa, len, hal }) + } + + pub fn pa(&self) -> u64 { + self.pa + } + + /// The kernel's pointer to byte `offset`. + /// + /// # Panics + /// + /// If `offset + size` exceeds the region. + pub fn ptr(&self, offset: usize, size: usize) -> *mut u8 { + assert!( + offset.checked_add(size).is_some_and(|end| end <= self.len), + "DMA access out of bounds" + ); + // Safety: within the region. + unsafe { self.va.as_ptr().add(offset) } + } +} + +impl Drop for DmaRegion { + fn drop(&mut self) { + // Safety: from `dma_alloc` with this many pages; see the type's docs + // for device access. + unsafe { self.hal.dma_dealloc(self.va, self.len / PAGE_SIZE) }; + } +} + +/// Device registers, mapped uncacheable. Accesses are volatile and +/// bounds-checked; offsets come from the driver, lengths from the device, +/// checked when the region is created. +#[derive(Clone, Copy, Debug)] +pub struct Mmio { + base: NonNull, + len: usize, +} + +// Safety: device registers have no Rust-visible state; drivers serialize +// their protocols themselves. +unsafe impl Send for Mmio {} +// Safety: as above. +unsafe impl Sync for Mmio {} + +macro_rules! mmio_accessors { + ($($read:ident $write:ident $ty:ty;)*) => {$( + pub fn $read(&self, offset: usize) -> $ty { + // Safety: in bounds and aligned (checked); a device register. + unsafe { self.at::<$ty>(offset).read_volatile() } + } + + pub fn $write(&self, offset: usize, value: $ty) { + // Safety: as above. + unsafe { self.at::<$ty>(offset).write_volatile(value) } + } + )*}; +} + +impl Mmio { + /// # Safety + /// + /// `base..base + len` must be device registers mapped uncacheable for the + /// result's lifetime and not used as memory by anything else. + pub unsafe fn new(base: NonNull, len: usize) -> Self { + Self { base, len } + } + + pub(crate) fn len(&self) -> usize { + self.len + } + + /// Device-supplied offsets are validated before they reach here, so an + /// out-of-bounds or misaligned access is a driver bug. + fn at(&self, offset: usize) -> *mut T { + assert!( + offset + .checked_add(size_of::()) + .is_some_and(|end| end <= self.len), + "MMIO access at {offset:#x} outside {:#x} bytes", + self.len + ); + // Safety: within the mapping. + let ptr: *mut T = unsafe { self.base.add(offset).cast().as_ptr() }; + // The address, not just the offset: a structure may start anywhere. + assert!(ptr.is_aligned(), "misaligned MMIO access at {ptr:p}"); + ptr + } + + mmio_accessors! { + read_u8 write_u8 u8; + read_u16 write_u16 u16; + read_u32 write_u32 u32; + } +} + +/// # Safety +/// +/// `dma_alloc` must return `pages` physically contiguous pages at `pa`, +/// mapped writable at the returned address and exclusively owned until +/// `dma_dealloc`. `map_mmio` must uphold [`Mmio::new`]'s contract. +pub unsafe trait Hal: Sync { + /// `None` when out of memory. + fn dma_alloc(&self, pages: usize) -> Option<(NonNull, u64)>; + + /// # Safety + /// + /// From `dma_alloc` with the same `pages`; no device accesses it. + unsafe fn dma_dealloc(&self, va: NonNull, pages: usize); + + /// Maps the device registers at physical `pa..pa + len` for the kernel's + /// lifetime; `None` if they cannot be (e.g., they overlap RAM). + /// + /// # Safety + /// + /// `pa..pa + len` must be device registers, e.g., within a BAR. + unsafe fn map_mmio(&self, pa: u64, len: usize) -> Option; +} + +#[cfg(test)] +mod tests { + use super::Mmio; + use core::ptr::NonNull; + + #[test] + #[should_panic(expected = "misaligned")] + fn misaligned_address_is_refused() { + let mut memory = [0u64; 4]; + // Safety: plain memory, alive for the test. + let mmio = unsafe { Mmio::new(NonNull::from(&mut memory).cast::().add(2), 30) }; + mmio.read_u32(0); + } +} diff --git a/litebox_hal/src/interrupt.rs b/litebox_hal/src/interrupt.rs index 23e234da77..66d9531ac7 100644 --- a/litebox_hal/src/interrupt.rs +++ b/litebox_hal/src/interrupt.rs @@ -1,24 +1,58 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Interrupt controllers: the legacy 8259 PICs. +//! Interrupt controllers: the local APIC, for device interrupts (MSI) and +//! the timer, in x2APIC mode (as on LVBS) where the CPU has it, else in +//! xAPIC mode (e.g., QEMU's TCG); the legacy 8259 PICs are only masked. + +use crate::clock::{Clock, now}; +use crate::dma::{Hal, Mmio}; +use x86_64::instructions::port::Port; +use x86_64::registers::model_specific::Msr; const LEGACY_PIC_VECTOR_BASE: u8 = 0x20; -/// PIC IRQ7/IRQ15 can arrive while masked; SeaBIOS leaves LAPIC SVR at 0xff. +const MASTER_CMD: u16 = 0x20; +const MASTER_DATA: u16 = 0x21; +const SLAVE_CMD: u16 = 0xa0; +const SLAVE_DATA: u16 = 0xa1; + +const IA32_APIC_BASE: u32 = 0x1b; +const IA32_APIC_BASE_EN: u64 = 1 << 11; +const IA32_APIC_BASE_EXTD: u64 = 1 << 10; +const IA32_TSC_DEADLINE: u32 = 0x6e0; + +/// Local APIC registers, as xAPIC offsets; x2APIC MSR `0x800 + offset / 16`. +mod reg { + pub const ID: usize = 0x20; + pub const EOI: usize = 0xb0; + pub const SVR: usize = 0xf0; + pub const LVT_TIMER: usize = 0x320; + pub const TIMER_INITIAL_COUNT: usize = 0x380; + pub const TIMER_CURRENT_COUNT: usize = 0x390; + pub const TIMER_DIVIDE: usize = 0x3e0; +} + +const SVR_ENABLE: u32 = 1 << 8; +const LVT_TIMER_MASKED: u32 = 1 << 16; +const LVT_TIMER_TSC_DEADLINE: u32 = 0b10 << 17; +/// Divide the timer's clock by 16. +const TIMER_DIVIDE_BY_16: u32 = 0b0011; +const TIMER_CALIBRATION_MS: u64 = 10; + +/// The local APIC's spurious vector; needs no EOI. +const APIC_SPURIOUS_VECTOR: u8 = 0xff; + +/// PIC IRQ7/IRQ15 can arrive while masked, and the local APIC raises its +/// spurious vector. pub const SPURIOUS_VECTORS: [u8; 3] = [ LEGACY_PIC_VECTOR_BASE + 7, LEGACY_PIC_VECTOR_BASE + 15, - 0xff, + APIC_SPURIOUS_VECTOR, ]; /// Must precede user entry (IF=1). PVH's PIC vectors overlap CPU exceptions. pub fn init_legacy_pics() { - use x86_64::instructions::port::Port; - const MASTER_CMD: u16 = 0x20; - const MASTER_DATA: u16 = 0x21; - const SLAVE_CMD: u16 = 0xa0; - const SLAVE_DATA: u16 = 0xa1; const ICW1_INIT_ICW4: u8 = 0x11; // edge-triggered, cascade, ICW4 follows const ICW4_8086: u8 = 0x01; // Safety: the standard 8259 initialization sequence; nothing else drives @@ -36,3 +70,169 @@ pub fn init_legacy_pics() { Port::::new(SLAVE_DATA).write(0xff); } } + +/// An MSI message: a write of `data` to `address` raises the interrupt. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MsiMessage { + pub address: u64, + pub data: u32, +} + +/// Where a device interrupt goes: `vector` on the CPU with local APIC ID +/// `apic_id`. Each delivery mechanism derives its own form (e.g., an MSI +/// message). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InterruptTarget { + pub apic_id: u32, + pub vector: u8, +} + +impl InterruptTarget { + /// As an MSI message (fixed delivery, edge); `None` if the APIC ID does + /// not fit an MSI address, which without interrupt remapping holds 8 + /// bits. + pub fn msi_message(self) -> Option { + let id = u8::try_from(self.apic_id).ok()?; + Some(MsiMessage { + address: 0xfee0_0000 | (u64::from(id) << 12), + data: u32::from(self.vector), + }) + } +} + +/// This CPU's local APIC, enabled, with its timer raising a vector when +/// armed. +pub struct LocalApic { + /// The xAPIC registers; `None` in x2APIC mode. + xapic: Option, + /// Timer ticks per TSC cycle, as (ticks, cycles); `None` for the + /// TSC-deadline timer. + one_shot: Option<(u64, u64)>, +} + +fn has_cpu_feature(ecx_bit: u32) -> bool { + core::arch::x86_64::__cpuid_count(1, 0).ecx & (1 << ecx_bit) != 0 +} + +impl LocalApic { + /// Enables the local APIC and sets its timer to raise `timer_vector`, + /// disarmed. Calibrates the timer against `clock` if it has no + /// TSC-deadline mode. + /// + /// # Panics + /// + /// If the xAPIC registers cannot be mapped. + pub fn init(hal: &dyn Hal, timer_vector: u8, clock: Clock) -> Self { + // The local APIC belongs to this kernel, and interrupts are disabled. + // The SDM requires enabling xAPIC before x2APIC. + let mut base = Msr::new(IA32_APIC_BASE); + // Safety: see above. + let value = unsafe { base.read() } | IA32_APIC_BASE_EN; + let xapic = if has_cpu_feature(21) { + // Safety: see above. + unsafe { + base.write(value); + base.write(value | IA32_APIC_BASE_EXTD); + } + None + } else { + // Safety: see above. + unsafe { base.write(value) }; + // Safety: the APIC base names the local APIC's registers. + let registers = unsafe { hal.map_mmio(value & 0xf_ffff_f000, 0x1000) }; + Some(registers.expect("the local APIC's registers can be mapped")) + }; + let mut apic = Self { + xapic, + one_shot: None, + }; + let svr = apic.read(reg::SVR); + apic.write(reg::SVR, svr | SVR_ENABLE | u32::from(APIC_SPURIOUS_VECTOR)); + if has_cpu_feature(24) { + apic.write( + reg::LVT_TIMER, + LVT_TIMER_TSC_DEADLINE | u32::from(timer_vector), + ); + } else { + apic.write(reg::TIMER_DIVIDE, TIMER_DIVIDE_BY_16); + apic.write(reg::LVT_TIMER, LVT_TIMER_MASKED | u32::from(timer_vector)); + apic.write(reg::TIMER_INITIAL_COUNT, u32::MAX); + let start = now(); + let cycles = TIMER_CALIBRATION_MS * clock.khz(); + while now() - start < cycles {} + let ticks = u64::from(u32::MAX - apic.read(reg::TIMER_CURRENT_COUNT)); + apic.write(reg::TIMER_INITIAL_COUNT, 0); + apic.write(reg::LVT_TIMER, u32::from(timer_vector)); + apic.one_shot = Some((ticks.max(1), cycles)); + } + litebox_util_log::info!( + mode:% = if apic.xapic.is_some() { "xAPIC" } else { "x2APIC" }, + timer:% = if apic.one_shot.is_some() { "one-shot" } else { "TSC-deadline" }; + "local APIC" + ); + apic + } + + /// x2APIC register `register`'s MSR. + fn msr(register: usize) -> Msr { + Msr::new(0x800 + u32::try_from(register >> 4).expect("small offsets")) + } + + fn read(&self, register: usize) -> u32 { + match self.xapic { + Some(registers) => registers.read_u32(register), + // Safety: reading a local APIC register has no side effects; the + // registers used are 32 bits. + #[expect(clippy::cast_possible_truncation, reason = "32-bit registers")] + None => unsafe { Self::msr(register).read() as u32 }, + } + } + + fn write(&self, register: usize, value: u32) { + match self.xapic { + Some(registers) => registers.write_u32(register, value), + // Safety: callers program the local APIC, which this kernel owns. + None => unsafe { Self::msr(register).write(u64::from(value)) }, + } + } + + /// For interrupt handlers: ends the interrupt. + pub fn end_of_interrupt(&self) { + self.write(reg::EOI, 0); + } + + /// Interrupts raising `vector` on this CPU. + pub fn target(&self, vector: u8) -> InterruptTarget { + let apic_id = match self.xapic { + Some(_) => self.read(reg::ID) >> 24, + None => self.read(reg::ID), + }; + InterruptTarget { apic_id, vector } + } + + /// Raises the timer's vector once, when [`now`] reaches `deadline` (at + /// once if it has; without TSC-deadline mode, a distant deadline may come + /// early); replaces a pending deadline. + pub fn arm_timer(&self, deadline: u64) { + match self.one_shot { + // Safety: programs this CPU's timer; zero would disarm. + None => unsafe { Msr::new(IA32_TSC_DEADLINE).write(deadline.max(1)) }, + Some((ticks, cycles)) => { + let delta = u128::from(deadline.saturating_sub(now())); + let count = delta * u128::from(ticks) / u128::from(cycles); + self.write( + reg::TIMER_INITIAL_COUNT, + u32::try_from(count.max(1)).unwrap_or(u32::MAX), + ); + } + } + } + + pub fn disarm_timer(&self) { + match self.one_shot { + // Safety: programs this CPU's timer. + None => unsafe { Msr::new(IA32_TSC_DEADLINE).write(0) }, + Some(_) => self.write(reg::TIMER_INITIAL_COUNT, 0), + } + } +} diff --git a/litebox_hal/src/lib.rs b/litebox_hal/src/lib.rs index 12b88bc4db..a153d674da 100644 --- a/litebox_hal/src/lib.rs +++ b/litebox_hal/src/lib.rs @@ -3,20 +3,29 @@ //! Hardware abstraction layer for LiteBox kernel-mode runners: what a VM //! cannot do by itself and gets from its (virtual) hardware or hypervisor. -//! Each module states the hardware it assumes. Runners choose and wire the -//! sources. +//! Each module states the hardware it assumes. [`devices`] puts devices +//! behind device-neutral types; runners choose and wire the sources. #![no_std] #![warn(clippy::undocumented_unsafe_blocks)] +extern crate alloc; + #[cfg(target_arch = "x86_64")] pub mod clock; #[cfg(target_arch = "x86_64")] -pub mod console; +pub mod devices; +pub mod dma; #[cfg(target_arch = "x86_64")] pub mod interrupt; #[cfg(target_arch = "x86_64")] +pub mod pci; +#[cfg(target_arch = "x86_64")] pub mod power; pub mod prk; #[cfg(target_arch = "x86_64")] pub mod rng; +#[cfg(target_arch = "x86_64")] +pub mod serial; +#[cfg(target_arch = "x86_64")] +pub mod virtio; diff --git a/litebox_hal/src/pci.rs b/litebox_hal/src/pci.rs new file mode 100644 index 0000000000..0b8ecb39fd --- /dev/null +++ b/litebox_hal/src/pci.rs @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! PCI: configuration space through the legacy mechanism #1 ports +//! (`0xCF8`/`0xCFC`), on bus 0, and MSI-X. Firmware (SeaBIOS under QEMU) has +//! already assigned BARs. + +use crate::dma::Hal; +use crate::interrupt::MsiMessage; +use x86_64::instructions::port::Port; + +const CONFIG_ADDRESS: u16 = 0xcf8; +const CONFIG_DATA: u16 = 0xcfc; + +const VENDOR_ID: u8 = 0x00; +const DEVICE_ID: u8 = 0x02; +const SUBSYSTEM_ID: u8 = 0x2e; +const COMMAND: u8 = 0x04; +const STATUS: u8 = 0x06; +const HEADER_TYPE: u8 = 0x0e; +const CAPABILITIES_POINTER: u8 = 0x34; +const STATUS_CAPABILITIES: u16 = 1 << 4; +/// Capabilities live past the standard header. +const FIRST_CAPABILITY: u8 = 0x40; +const BAR0: u8 = 0x10; + +const CAP_MSIX: u8 = 0x11; +const MSIX_CONTROL_ENABLE: u16 = 1 << 15; +const MSIX_CONTROL_FUNCTION_MASK: u16 = 1 << 14; +const MSIX_ENTRY_SIZE: usize = 16; + +/// `COMMAND` bits. +pub const COMMAND_IO_SPACE: u16 = 1 << 0; +pub const COMMAND_MEMORY_SPACE: u16 = 1 << 1; +pub const COMMAND_BUS_MASTER: u16 = 1 << 2; +pub const COMMAND_INTX_DISABLE: u16 = 1 << 10; + +/// A function on bus 0. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Function { + pub device: u8, + pub function: u8, +} + +impl core::fmt::Display for Function { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "00:{:02x}.{}", self.device, self.function) + } +} + +impl Function { + fn select(self, offset: u8) { + let address = 0x8000_0000 + | (u32::from(self.device & 0x1f) << 11) + | (u32::from(self.function & 0x7) << 8) + | u32::from(offset & 0xfc); + // Safety: selects a configuration register; no memory side effects. + // Nothing else in the kernel accesses PCI configuration space, and + // callers run with interrupts disabled, so the address/data pair is + // not interleaved. + unsafe { Port::::new(CONFIG_ADDRESS).write(address) }; + } + + pub fn read_u32(self, offset: u8) -> u32 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA).read() } + } + + pub fn read_u16(self, offset: u8) -> u16 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 2)).read() } + } + + pub fn read_u8(self, offset: u8) -> u8 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 3)).read() } + } + + /// # Safety + /// + /// The write must not move a BAR or enable DMA into memory the caller + /// does not own. + pub unsafe fn write_u32(self, offset: u8, value: u32) { + self.select(offset); + // Safety: forwarded to the caller. + unsafe { Port::::new(CONFIG_DATA).write(value) }; + } + + pub fn vendor_id(self) -> u16 { + self.read_u16(VENDOR_ID) + } + + pub fn device_id(self) -> u16 { + self.read_u16(DEVICE_ID) + } + + pub fn subsystem_id(self) -> u16 { + self.read_u16(SUBSYSTEM_ID) + } + + pub fn command(self) -> u16 { + self.read_u16(COMMAND) + } + + /// A 16-bit write, leaving the neighboring register (e.g., `STATUS`, + /// write-1-to-clear) alone. + /// + /// # Safety + /// + /// As for [`Self::write_u32`]. + pub unsafe fn write_u16(self, offset: u8, value: u16) { + self.select(offset); + // Safety: forwarded to the caller. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 2)).write(value) }; + } + + /// # Safety + /// + /// Enabling bus mastering lets the device DMA wherever the driver points + /// it. + pub unsafe fn set_command(self, command: u16) { + // Safety: forwarded to the caller. + unsafe { self.write_u16(COMMAND, command) }; + } + + /// Memory BAR `index` (with its upper half, if 64-bit), sized; `None` if + /// it is not a memory BAR or is unassigned. Sizing briefly disables the + /// function's decoding. + /// + /// # Safety + /// + /// Nothing may access the function's BARs meanwhile. + pub unsafe fn memory_bar(self, index: u8) -> Option { + if index >= 6 { + return None; + } + let offset = BAR0 + 4 * index; + let low = self.read_u32(offset); + let wide = (low >> 1) & 3 == 2; + if low & 1 != 0 || (wide && index == 5) { + return None; + } + let high = if wide { self.read_u32(offset + 4) } else { 0 }; + let command = self.command(); + // Safety: decoding is off while the BAR reads back its size mask, and + // both halves are restored before it is turned back on. + let (low_mask, high_mask) = unsafe { + self.set_command(command & !(COMMAND_IO_SPACE | COMMAND_MEMORY_SPACE)); + self.write_u32(offset, u32::MAX); + let low_mask = self.read_u32(offset); + self.write_u32(offset, low); + let high_mask = if wide { + self.write_u32(offset + 4, u32::MAX); + let mask = self.read_u32(offset + 4); + self.write_u32(offset + 4, high); + mask + } else { + u32::MAX + }; + self.set_command(command); + (low_mask, high_mask) + }; + let mask = (u64::from(high_mask) << 32) | u64::from(low_mask & !0xf); + let size = (!mask).wrapping_add(1); + let address = (u64::from(high) << 32) | u64::from(low & !0xf); + (size != 0 && address != 0 && address.checked_add(size).is_some()) + .then_some(MemoryBar { address, size }) + } + + /// Calls `f` with the offset and ID of each capability. The list comes + /// from the device: a malformed or cyclic one ends the walk. + pub fn for_each_capability(self, mut f: impl FnMut(u8, u8)) { + if self.read_u16(STATUS) & STATUS_CAPABILITIES == 0 { + return; + } + let mut offset = self.read_u8(CAPABILITIES_POINTER) & 0xfc; + // At most this many fit in configuration space. + for _ in 0..(256 - usize::from(FIRST_CAPABILITY)) / 4 { + if offset < FIRST_CAPABILITY { + return; + } + f(offset, self.read_u8(offset)); + offset = self.read_u8(offset + 1) & 0xfc; + } + } + + /// Routes every MSI-X vector of the function to `message` and enables + /// MSI-X (which disables INTx); `false` if it has no usable MSI-X. + /// + /// # Safety + /// + /// Nothing may access the function's BARs meanwhile, and `message` must + /// raise an interrupt the kernel handles. + pub unsafe fn enable_msix(self, hal: &dyn Hal, message: MsiMessage) -> bool { + let mut msix = None; + self.for_each_capability(|offset, id| { + if id == CAP_MSIX && offset <= 0xf4 { + msix.get_or_insert(offset); + } + }); + let Some(cap) = msix else { + return false; + }; + let control = self.read_u16(cap + 2); + let entries = usize::from(control & 0x7ff) + 1; + let table = self.read_u32(cap + 4); + // Safety: the caller does not access the BARs meanwhile. + let Some(bar) = (unsafe { self.memory_bar(table.to_le_bytes()[0] & 7) }) else { + return false; + }; + let (offset, len) = (u64::from(table & !7), (entries * MSIX_ENTRY_SIZE) as u64); + if offset + len > bar.size { + return false; + } + // Safety: the table is within the BAR, so device registers. + let Some(table) = + (unsafe { hal.map_mmio(bar.address + offset, entries * MSIX_ENTRY_SIZE) }) + else { + return false; + }; + // Safety: decodes the BAR firmware assigned, so the table writes + // below reach the device. + unsafe { self.set_command(self.command() | COMMAND_MEMORY_SPACE) }; + #[expect(clippy::cast_possible_truncation, reason = "split into words")] + for entry in (0..entries).map(|entry| entry * MSIX_ENTRY_SIZE) { + table.write_u32(entry, message.address as u32); + table.write_u32(entry + 4, (message.address >> 32) as u32); + table.write_u32(entry + 8, message.data); + table.write_u32(entry + 12, 0); // unmasked + } + // Safety: the vectors raise the caller's interrupt. + unsafe { + self.write_u16( + cap + 2, + (control | MSIX_CONTROL_ENABLE) & !MSIX_CONTROL_FUNCTION_MASK, + ); + } + true + } +} + +/// A memory BAR's physical range. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MemoryBar { + pub address: u64, + pub size: u64, +} + +/// The first function on bus 0 that `matches`. +pub fn find(matches: impl Fn(Function) -> bool) -> Option { + for device in 0..32 { + let first = Function { + device, + function: 0, + }; + if first.vendor_id() == 0xffff { + continue; + } + let multifunction = first.read_u8(HEADER_TYPE) & 0x80 != 0; + for function in 0..if multifunction { 8 } else { 1 } { + let candidate = Function { device, function }; + if matches(candidate) { + return Some(candidate); + } + } + } + None +} diff --git a/litebox_hal/src/console.rs b/litebox_hal/src/serial.rs similarity index 86% rename from litebox_hal/src/console.rs rename to litebox_hal/src/serial.rs index 8da1b619ff..65afd02d0e 100644 --- a/litebox_hal/src/console.rs +++ b/litebox_hal/src/serial.rs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Console: a 16550 UART on COM1. Output is dropped if no UART answers. +//! The serial port, for the kernel's own output: a 16550 UART on COM1. +//! Output is dropped if no UART answers. use core::fmt; use spin::{Mutex, Once}; @@ -77,15 +78,19 @@ impl ComPort { } } - fn write_string(&mut self, s: &str) { + fn write_bytes(&mut self, bytes: &[u8]) { if !self.available { return; } - for byte in s.bytes() { + for &byte in bytes { self.write_byte(byte); } } + + fn write_string(&mut self, s: &str) { + self.write_bytes(s.as_bytes()); + } } static COM_ONCE: Once> = Once::new(); @@ -114,6 +119,11 @@ pub fn print_str(s: &str) { com().lock().write_string(s); } +/// Non-printable bytes show as `0xfe`. +pub fn print_bytes(bytes: &[u8]) { + com().lock().write_bytes(bytes); +} + /// Panic may interrupt a locked or initializing console. Never wait for it /// or alias its mutable handle; emergency output may interleave. pub fn panic_print(args: core::fmt::Arguments) { @@ -129,9 +139,9 @@ pub fn panic_print(args: core::fmt::Arguments) { let _ = com.write_fmt(args); } -/// `println!` to the console. +/// `println!` to the serial port. #[macro_export] -macro_rules! console_println { - () => ($crate::console::print(format_args!("\n"))); - ($($arg:tt)*) => ($crate::console::print(format_args!("{}\n", format_args!($($arg)*)))); +macro_rules! serial_println { + () => ($crate::serial::print(format_args!("\n"))); + ($($arg:tt)*) => ($crate::serial::print(format_args!("{}\n", format_args!($($arg)*)))); } diff --git a/litebox_hal/src/virtio/console.rs b/litebox_hal/src/virtio/console.rs new file mode 100644 index 0000000000..5a287a8773 --- /dev/null +++ b/litebox_hal/src/virtio/console.rs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Virtio console (device type 3): QEMU's `virtio-serial-pci` with a +//! `virtconsole` on port 0, e.g.: +//! +//! ```text +//! -device virtio-serial-pci,id=vs0,disable-legacy=on +//! -chardev file,id=con0,path=out.txt,input-path=in.txt +//! -device virtconsole,chardev=con0,bus=vs0.0 +//! ``` +//! +//! Without `VIRTIO_CONSOLE_F_MULTIPORT` (not negotiated), the device is a +//! single port (port 0, where a `virtconsole` attaches) on queues 0 +//! (receive) and 1 (transmit), open without control-queue traffic. +//! +//! Neither direction blocks: [`VirtioConsole::write`] takes what free +//! transmit buffers hold, and [`VirtioConsole::read`] what has arrived. The +//! device interrupts when it returns buffers in either direction. + +use super::Error; +use super::pci::PciTransport; +use super::queue::{Buffer, VirtQueue}; +use crate::dma::{DmaRegion, Hal}; +use crate::interrupt::MsiMessage; +use alloc::collections::VecDeque; +use alloc::vec; +use alloc::vec::Vec; + +const DEVICE_TYPE: u16 = 3; +const RECEIVE_QUEUE: u16 = 0; +const TRANSMIT_QUEUE: u16 = 1; + +const RECEIVE_BUFFERS: u16 = 16; +const RECEIVE_BUFFER_SIZE: usize = 256; +const TRANSMIT_BUFFERS: u16 = 32; +const TRANSMIT_BUFFER_SIZE: usize = 512; + +/// Fixed-size buffers in one DMA region, each a one-descriptor chain while +/// the device has it. +struct Buffers { + memory: DmaRegion, + size: usize, + /// By queue head: the buffer the device has. + by_head: Vec>, + free: Vec, +} + +impl Buffers { + fn new( + hal: &'static dyn Hal, + queue: &VirtQueue, + count: u16, + size: usize, + ) -> Result { + let count = count.min(queue.size()); + Ok(Self { + memory: DmaRegion::new(hal, usize::from(count) * size).ok_or(Error::OutOfMemory)?, + size, + by_head: vec![None; usize::from(queue.size())], + free: (0..count).rev().collect(), + }) + } + + fn ptr(&self, buffer: u16, offset: usize, len: usize) -> *mut u8 { + assert!(offset + len <= self.size); + self.memory + .ptr(usize::from(buffer) * self.size + offset, len) + } + + /// Gives a free buffer (`len` bytes of it) to the device. + fn post(&mut self, queue: &mut VirtQueue, buffer: u16, len: usize, device_writes: bool) { + let chain = [Buffer { + pa: self.memory.pa() + (usize::from(buffer) * self.size) as u64, + len: u32::try_from(len.min(self.size)).expect("buffers are small"), + device_writes, + }]; + // Safety: the buffer is ours (free) and DMA memory that lives as + // long as the console; it is not accessed until the device returns + // it. + let head = unsafe { queue.add(&chain) }.expect("a descriptor per buffer"); + self.by_head[usize::from(head)] = Some(buffer); + } + + /// The next buffer the device returned, with its length clamped. + fn take_used(&mut self, queue: &mut VirtQueue) -> Option<(u16, usize)> { + let (head, len) = queue.pop_used()?; + let buffer = self.by_head[usize::from(head)] + .take() + .expect("in-flight chains are posted buffers"); + Some(( + buffer, + usize::try_from(len).map_or(self.size, |len| len.min(self.size)), + )) + } +} + +/// A live console. Not internally synchronized. +pub struct VirtioConsole { + transport: PciTransport, + receive: VirtQueue, + receive_buffers: Buffers, + transmit: VirtQueue, + transmit_buffers: Buffers, + /// Filled receive buffers not yet read, as (buffer, length), oldest + /// first; `received_offset` bytes of the first are read. + received: VecDeque<(u16, usize)>, + received_offset: usize, +} + +impl VirtioConsole { + /// The first virtio console on bus 0, brought up, interrupting with + /// `interrupt` (if it can; see [`PciTransport::interrupts`]); `None` if + /// there is none. + /// + /// # Errors + /// + /// The device cannot be driven. + pub fn probe( + hal: &'static dyn Hal, + interrupt: Option, + ) -> Option> { + let function = super::find_pci_device(DEVICE_TYPE)?; + Some( + PciTransport::new(hal, function, interrupt) + .and_then(|transport| Self::bring_up(hal, transport)), + ) + } + + fn bring_up(hal: &'static dyn Hal, transport: PciTransport) -> Result { + transport.init(0)?; + let setup = || -> Result<_, Error> { + let receive = transport.setup_queue(hal, RECEIVE_QUEUE, RECEIVE_BUFFERS)?; + let transmit = transport.setup_queue(hal, TRANSMIT_QUEUE, TRANSMIT_BUFFERS)?; + let receive_buffers = + Buffers::new(hal, &receive, RECEIVE_BUFFERS, RECEIVE_BUFFER_SIZE)?; + let transmit_buffers = + Buffers::new(hal, &transmit, TRANSMIT_BUFFERS, TRANSMIT_BUFFER_SIZE)?; + Ok((receive, receive_buffers, transmit, transmit_buffers)) + }; + let (mut receive, mut receive_buffers, transmit, transmit_buffers) = + setup().inspect_err(|_| transport.fail())?; + while let Some(buffer) = receive_buffers.free.pop() { + receive_buffers.post(&mut receive, buffer, RECEIVE_BUFFER_SIZE, true); + } + transport.start(); + transport.notify(&receive); + Ok(Self { + transport, + receive, + receive_buffers, + transmit, + transmit_buffers, + received: VecDeque::with_capacity(usize::from(RECEIVE_BUFFERS)), + received_offset: 0, + }) + } + + pub fn transport(&self) -> &PciTransport { + &self.transport + } + + /// Queues as much of `bytes` as free transmit buffers hold; returns how + /// much. Zero means every buffer is still with the device. + pub fn write(&mut self, bytes: &[u8]) -> usize { + while let Some((buffer, _)) = self.transmit_buffers.take_used(&mut self.transmit) { + self.transmit_buffers.free.push(buffer); + } + let mut written = 0; + while written < bytes.len() { + let Some(buffer) = self.transmit_buffers.free.pop() else { + break; + }; + let chunk = &bytes[written..]; + let chunk = &chunk[..chunk.len().min(TRANSMIT_BUFFER_SIZE)]; + let destination = self.transmit_buffers.ptr(buffer, 0, chunk.len()); + // Safety: a free buffer, which the device does not access. + unsafe { core::ptr::copy_nonoverlapping(chunk.as_ptr(), destination, chunk.len()) }; + self.transmit_buffers + .post(&mut self.transmit, buffer, chunk.len(), false); + written += chunk.len(); + } + if written != 0 { + self.transport.notify(&self.transmit); + } + written + } + + /// Whether [`Self::read`] would return data. + pub fn has_input(&mut self) -> bool { + self.collect_received(); + !self.received.is_empty() + } + + /// Copies received bytes into `output`; returns how many. + pub fn read(&mut self, output: &mut [u8]) -> usize { + self.collect_received(); + let mut read = 0; + let mut reposted = false; + while read < output.len() { + let Some(&(buffer, len)) = self.received.front() else { + break; + }; + let n = (len - self.received_offset).min(output.len() - read); + let source = self.receive_buffers.ptr(buffer, self.received_offset, n); + // Safety: the device returned the buffer; `n` bytes are in it. + unsafe { core::ptr::copy_nonoverlapping(source, output[read..].as_mut_ptr(), n) }; + read += n; + self.received_offset += n; + if self.received_offset == len { + self.received.pop_front(); + self.received_offset = 0; + self.receive_buffers + .post(&mut self.receive, buffer, RECEIVE_BUFFER_SIZE, true); + reposted = true; + } + } + if reposted { + self.transport.notify(&self.receive); + } + read + } + + fn collect_received(&mut self) { + let mut reposted = false; + while let Some((buffer, len)) = self.receive_buffers.take_used(&mut self.receive) { + if len == 0 { + self.receive_buffers + .post(&mut self.receive, buffer, RECEIVE_BUFFER_SIZE, true); + reposted = true; + } else { + self.received.push_back((buffer, len)); + } + } + if reposted { + self.transport.notify(&self.receive); + } + } +} diff --git a/litebox_hal/src/virtio/mod.rs b/litebox_hal/src/virtio/mod.rs new file mode 100644 index 0000000000..bf0ac909d4 --- /dev/null +++ b/litebox_hal/src/virtio/mod.rs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Virtio 1.x devices over PCI ([`pci::PciTransport`]): the modern +//! interface, with its structures in memory BARs (mapped through +//! [`crate::dma::Hal`]) and interrupts by MSI-X. Split virtqueues +//! ([`queue`]). +//! +//! Devices: [`console`]. The transport and queues are device-independent, +//! for further devices (e.g., file systems and networking for the broker). +//! +//! Devices are not trusted: the capability list, BAR ranges, queue sizes and +//! used-ring entries are checked, and lengths the device reports are clamped +//! by the driver before use. + +pub mod console; +pub mod pci; +pub mod queue; + +const PCI_VENDOR: u16 = 0x1af4; + +/// The first virtio function of type `device_type` on bus 0. Modern-only +/// device IDs are 0x1040 + type; transitional ones (0x1000..=0x103f) give +/// the type in the subsystem ID. +pub fn find_pci_device(device_type: u16) -> Option { + crate::pci::find(|function| { + let device_id = function.device_id(); + function.vendor_id() == PCI_VENDOR + && (device_id == 0x1040 + device_type + || ((0x1000..=0x103f).contains(&device_id) + && function.subsystem_id() == device_type)) + }) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// The device lacks a required structure (common or notify). + MissingStructure(&'static str), + /// A structure is outside its BAR, too short, or not in a memory BAR. + BadStructure(&'static str), + /// The platform could not map a structure. + MapFailed, + ResetTimedOut, + /// No `VIRTIO_F_VERSION_1`. + NotModern, + /// The device cleared `FEATURES_OK`. + FeaturesRejected, + QueueUnavailable(u16), + /// The queue's notification address is outside the notify structure. + BadNotifyOffset(u16), + OutOfMemory, +} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::MissingStructure(what) => write!(f, "no {what} structure"), + Self::BadStructure(what) => write!(f, "malformed {what} structure"), + Self::MapFailed => f.write_str("cannot map device registers"), + Self::ResetTimedOut => f.write_str("reset timed out"), + Self::NotModern => f.write_str("VIRTIO_F_VERSION_1 not offered"), + Self::FeaturesRejected => f.write_str("features rejected"), + Self::QueueUnavailable(queue) => write!(f, "queue {queue} unavailable"), + Self::BadNotifyOffset(queue) => write!(f, "queue {queue} notify offset out of range"), + Self::OutOfMemory => f.write_str("out of DMA memory"), + } + } +} diff --git a/litebox_hal/src/virtio/pci.rs b/litebox_hal/src/virtio/pci.rs new file mode 100644 index 0000000000..065269db51 --- /dev/null +++ b/litebox_hal/src/virtio/pci.rs @@ -0,0 +1,310 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! The modern virtio-PCI transport: structures located by vendor-specific +//! capabilities, in memory BARs; interrupts by MSI-X. + +use super::Error; +use super::queue::VirtQueue; +use crate::dma::{Hal, Mmio}; +use crate::interrupt::MsiMessage; +use crate::pci::{self, Function, MemoryBar}; + +const CAP_VENDOR_SPECIFIC: u8 = 0x09; + +/// `VIRTIO_F_VERSION_1`: the modern interface; always negotiated. +const F_VERSION_1: u64 = 1 << 32; + +/// `virtio_pci_cap` fields, from the capability's offset. +mod cap { + pub const CFG_TYPE: u8 = 3; + pub const BAR: u8 = 4; + pub const OFFSET: u8 = 8; + pub const LENGTH: u8 = 12; + /// `virtio_pci_notify_cap` only. + pub const NOTIFY_OFF_MULTIPLIER: u8 = 16; + /// Of a `virtio_pci_notify_cap`, the longest one read. + pub const LEN: u8 = 20; + + pub const COMMON: u8 = 1; + pub const NOTIFY: u8 = 2; + pub const DEVICE: u8 = 4; +} + +/// `virtio_pci_common_cfg` fields. +mod common { + pub const DEVICE_FEATURE_SELECT: usize = 0x00; + pub const DEVICE_FEATURE: usize = 0x04; + pub const DRIVER_FEATURE_SELECT: usize = 0x08; + pub const DRIVER_FEATURE: usize = 0x0c; + pub const DEVICE_STATUS: usize = 0x14; + pub const QUEUE_SELECT: usize = 0x16; + pub const QUEUE_SIZE: usize = 0x18; + pub const QUEUE_MSIX_VECTOR: usize = 0x1a; + pub const QUEUE_ENABLE: usize = 0x1c; + pub const QUEUE_NOTIFY_OFF: usize = 0x1e; + pub const QUEUE_DESC: usize = 0x20; + pub const QUEUE_DRIVER: usize = 0x28; + pub const QUEUE_DEVICE: usize = 0x30; + pub const LEN: usize = 0x38; +} + +mod status { + pub const ACKNOWLEDGE: u8 = 1; + pub const DRIVER: u8 = 2; + pub const DRIVER_OK: u8 = 4; + pub const FEATURES_OK: u8 = 8; + pub const FAILED: u8 = 0x80; +} + +const RESET_POLL_LIMIT: u32 = 1_000_000; + +/// A structure's location, from its capability. +#[derive(Clone, Copy)] +struct Location { + bar: u8, + offset: u32, + length: u32, +} + +/// The device's capabilities: the first of each structure type, as the +/// specification asks drivers to prefer. +#[derive(Default)] +struct Locations { + common: Option, + notify: Option<(Location, u32)>, + device: Option, +} + +impl Locations { + fn read(function: Function) -> Self { + let mut found = Self::default(); + function.for_each_capability(|offset, id| { + // Every field read must be inside configuration space. + if id != CAP_VENDOR_SPECIFIC || offset.checked_add(cap::LEN).is_none() { + return; + } + let location = Location { + bar: function.read_u8(offset + cap::BAR), + offset: function.read_u32(offset + cap::OFFSET), + length: function.read_u32(offset + cap::LENGTH), + }; + match function.read_u8(offset + cap::CFG_TYPE) { + cap::COMMON => { + found.common.get_or_insert(location); + } + cap::NOTIFY => { + let multiplier = function.read_u32(offset + cap::NOTIFY_OFF_MULTIPLIER); + found.notify.get_or_insert((location, multiplier)); + } + cap::DEVICE => { + found.device.get_or_insert(location); + } + _ => {} + } + }); + found + } +} + +/// A virtio PCI function's mapped structures. +pub struct PciTransport { + function: Function, + common: Mmio, + notify: Mmio, + notify_off_multiplier: u32, + device: Option, + /// Whether the device interrupts (by MSI-X); else it is polled. + interrupts: bool, +} + +impl PciTransport { + /// Maps `function`'s structures, enables its memory decoding and bus + /// mastering, and sends its interrupts as `interrupt` if it has MSI-X. + /// + /// # Errors + /// + /// A missing or malformed structure, or one that cannot be mapped. + pub fn new( + hal: &dyn Hal, + function: Function, + interrupt: Option, + ) -> Result { + let locations = Locations::read(function); + let mut bars = [None::; 6]; + let mut map = |location: Location, what, min_len: usize| -> Result { + let bad = Error::BadStructure(what); + let slot = bars.get_mut(usize::from(location.bar)).ok_or(bad)?; + let bar = match *slot { + Some(bar) => bar, + // Safety: the driver does not access the BARs yet. + None => *slot.insert(unsafe { function.memory_bar(location.bar) }.ok_or(bad)?), + }; + let (offset, length) = (u64::from(location.offset), u64::from(location.length)); + // Fields are up to 4 bytes, naturally aligned; BARs are aligned + // to their size, so this aligns the addresses too. + if !offset.is_multiple_of(4) || length < min_len as u64 || offset + length > bar.size { + return Err(bad); + } + let length = usize::try_from(length).map_err(|_| bad)?; + // Safety: within the BAR, so device registers. + unsafe { hal.map_mmio(bar.address + offset, length) }.ok_or(Error::MapFailed) + }; + let common = locations.common.ok_or(Error::MissingStructure("common"))?; + let (notify, notify_off_multiplier) = + locations.notify.ok_or(Error::MissingStructure("notify"))?; + let common = map(common, "common", common::LEN)?; + let notify = map(notify, "notify", 2)?; + let device = locations + .device + .map(|device| map(device, "device", 0)) + .transpose()?; + // Safety: the driver does not access the BARs yet; the message is the + // caller's interrupt. + let interrupts = + interrupt.is_some_and(|message| unsafe { function.enable_msix(hal, message) }); + let command = (function.command() | pci::COMMAND_MEMORY_SPACE | pci::COMMAND_BUS_MASTER) + | pci::COMMAND_INTX_DISABLE; + // Safety: decodes the BARs firmware assigned; DMA only goes where the + // driver points the queues. + unsafe { function.set_command(command) }; + Ok(Self { + function, + common, + notify, + notify_off_multiplier, + device, + interrupts, + }) + } + + pub fn function(&self) -> Function { + self.function + } + + /// Whether the device interrupts (by MSI-X); else it must be polled. + pub fn interrupts(&self) -> bool { + self.interrupts + } + + /// The device-specific configuration, if any. + pub fn device_config(&self) -> Option { + self.device + } + + fn status(&self) -> u8 { + self.common.read_u8(common::DEVICE_STATUS) + } + + fn set_status(&self, status: u8) { + self.common.write_u8(common::DEVICE_STATUS, status); + } + + fn add_status(&self, bits: u8) { + self.set_status(self.status() | bits); + } + + /// Marks the device `FAILED`, e.g., after a failed initialization. + pub fn fail(&self) { + self.add_status(status::FAILED); + } + + /// Resets the device and negotiates `wanted` features (plus + /// `VIRTIO_F_VERSION_1`) as far as it offers them; returns those. Set up + /// queues next, then [`Self::start`]. + /// + /// # Errors + /// + /// The device did not reset, is not modern, or rejected the features. It + /// is left `FAILED`, except after a reset timeout. + pub fn init(&self, wanted: u64) -> Result { + self.set_status(0); + if !(0..RESET_POLL_LIMIT).any(|_| self.status() == 0) { + return Err(Error::ResetTimedOut); + } + self.add_status(status::ACKNOWLEDGE); + self.add_status(status::DRIVER); + let offered = (0..2u32).fold(0, |features, word| { + self.common.write_u32(common::DEVICE_FEATURE_SELECT, word); + features | u64::from(self.common.read_u32(common::DEVICE_FEATURE)) << (32 * word) + }); + if offered & F_VERSION_1 == 0 { + self.fail(); + return Err(Error::NotModern); + } + let accepted = offered & (wanted | F_VERSION_1); + for word in 0..2u32 { + self.common.write_u32(common::DRIVER_FEATURE_SELECT, word); + #[expect(clippy::cast_possible_truncation, reason = "one 32-bit word")] + self.common + .write_u32(common::DRIVER_FEATURE, (accepted >> (32 * word)) as u32); + } + self.add_status(status::FEATURES_OK); + if self.status() & status::FEATURES_OK == 0 { + self.fail(); + return Err(Error::FeaturesRejected); + } + Ok(accepted) + } + + /// Sets up and enables queue `index`, with at most `max_size` entries, + /// interrupting on MSI-X vector 0 if the device interrupts. + /// + /// # Errors + /// + /// The device lacks the queue, its notification address or vector is + /// unusable, or memory ran out. + pub fn setup_queue( + &self, + hal: &'static dyn Hal, + index: u16, + max_size: u16, + ) -> Result { + self.common.write_u16(common::QUEUE_SELECT, index); + let device_max = self.common.read_u16(common::QUEUE_SIZE); + // Split queues of a power-of-two size work with every device. + let size = device_max.min(max_size); + if size == 0 { + return Err(Error::QueueUnavailable(index)); + } + let size = 1 << size.ilog2(); + let notify_offset = usize::from(self.common.read_u16(common::QUEUE_NOTIFY_OFF)) + .checked_mul(usize::try_from(self.notify_off_multiplier).unwrap_or(usize::MAX)) + .filter(|&offset| offset.is_multiple_of(2) && offset + 2 <= self.notify.len()) + .ok_or(Error::BadNotifyOffset(index))?; + if self.interrupts { + self.common.write_u16(common::QUEUE_MSIX_VECTOR, 0); + // The device answers `NO_VECTOR` (0xffff) if it cannot. + if self.common.read_u16(common::QUEUE_MSIX_VECTOR) != 0 { + return Err(Error::QueueUnavailable(index)); + } + } + let queue = VirtQueue::new(hal, index, size, notify_offset)?; + let [desc, driver, device] = queue.addresses(); + self.common.write_u16(common::QUEUE_SIZE, size); + for (field, address) in [ + (common::QUEUE_DESC, desc), + (common::QUEUE_DRIVER, driver), + (common::QUEUE_DEVICE, device), + ] { + #[expect(clippy::cast_possible_truncation, reason = "split into words")] + { + self.common.write_u32(field, address as u32); + self.common.write_u32(field + 4, (address >> 32) as u32); + } + } + self.common.write_u16(common::QUEUE_ENABLE, 1); + Ok(queue) + } + + /// Lets the device use its queues. + pub fn start(&self) { + self.add_status(status::DRIVER_OK); + } + + /// Tells the device that `queue` has new available buffers. + pub fn notify(&self, queue: &VirtQueue) { + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + self.notify.write_u16(queue.notify_offset(), queue.index()); + } +} diff --git a/litebox_hal/src/virtio/queue.rs b/litebox_hal/src/virtio/queue.rs new file mode 100644 index 0000000000..683ad6f442 --- /dev/null +++ b/litebox_hal/src/virtio/queue.rs @@ -0,0 +1,282 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Split virtqueues. A request is a chain of buffers (device-readable, then +//! device-writable), identified by its head descriptor until the device +//! returns it. The driver keeps its own record of chains and the free list, +//! so a device that writes nonsense into the used ring cannot corrupt them. + +use super::Error; +use crate::dma::{DmaRegion, Hal}; +use alloc::vec; +use alloc::vec::Vec; +use core::sync::atomic::{Ordering, fence}; + +const DESC_F_NEXT: u16 = 1; +const DESC_F_WRITE: u16 = 2; +const DESC_SIZE: usize = 16; + +/// One buffer of a chain, in DMA memory. +#[derive(Clone, Copy, Debug)] +pub struct Buffer { + pub pa: u64, + pub len: u32, + /// The device writes it (else reads it). Device-writable buffers follow + /// all device-readable ones in a chain. + pub device_writes: bool, +} + +pub struct VirtQueue { + index: u16, + size: u16, + notify_offset: usize, + /// Descriptors, then the available ring; the used ring at `used`. + ring: DmaRegion, + avail: usize, + used: usize, + /// Private copies of each descriptor's `next`, threading the free list + /// and in-flight chains. + next: Vec, + /// By head: the length of an in-flight chain, else zero. + in_flight: Vec, + free_head: u16, + num_free: u16, + next_avail: u16, + last_used: u16, +} + +// Safety: the queue exclusively owns its ring. +unsafe impl Send for VirtQueue {} + +impl VirtQueue { + /// `size` must be a power of two; see `PciTransport::setup_queue`. + pub(super) fn new( + hal: &'static dyn Hal, + index: u16, + size: u16, + notify_offset: usize, + ) -> Result { + assert!(size.is_power_of_two()); + let n = usize::from(size); + // Alignments: descriptors 16, available ring 2, used ring 4. + let avail = DESC_SIZE * n; + let used = (avail + 6 + 2 * n).next_multiple_of(4); + let ring = DmaRegion::new(hal, used + 6 + 8 * n).ok_or(Error::OutOfMemory)?; + Ok(Self { + index, + size, + notify_offset, + ring, + avail, + used, + next: (1..=size).collect(), + in_flight: vec![0; n], + free_head: 0, + num_free: size, + next_avail: 0, + last_used: 0, + }) + } + + pub fn index(&self) -> u16 { + self.index + } + + pub fn size(&self) -> u16 { + self.size + } + + pub(super) fn notify_offset(&self) -> usize { + self.notify_offset + } + + /// The descriptor table's, available ring's, and used ring's addresses. + pub(super) fn addresses(&self) -> [u64; 3] { + let pa = self.ring.pa(); + [pa, pa + self.avail as u64, pa + self.used as u64] + } + + fn write(&self, offset: usize, value: T) { + // Safety: in bounds (checked by `ptr`) and naturally aligned (the + // layout keeps every field so); the driver's part of the ring. + unsafe { + self.ring + .ptr(offset, size_of::()) + .cast::() + .write_volatile(value); + } + } + + fn read(&self, offset: usize) -> T { + // Safety: as for `write`; the device's part is read volatile. + unsafe { + self.ring + .ptr(offset, size_of::()) + .cast::() + .read_volatile() + } + } + + /// Makes `chain` available to the device; returns its head, or `None` if + /// it is empty or there are not enough free descriptors. The device + /// learns of it at the next [`super::pci::PciTransport::notify`]. + /// + /// # Safety + /// + /// The buffers must stay valid DMA memory, accessed only as the device + /// permits, until the device returns the chain ([`Self::pop_used`]). + pub unsafe fn add(&mut self, chain: &[Buffer]) -> Option { + let len = u16::try_from(chain.len()).ok()?; + if len == 0 || len > self.num_free { + return None; + } + let head = self.free_head; + let mut id = head; + for (i, buffer) in chain.iter().enumerate() { + let desc = usize::from(id) * DESC_SIZE; + let next = self.next[usize::from(id)]; + let mut flags = if buffer.device_writes { + DESC_F_WRITE + } else { + 0 + }; + if i + 1 < chain.len() { + flags |= DESC_F_NEXT; + } + self.write(desc, buffer.pa); + self.write(desc + 8, buffer.len); + self.write(desc + 12, flags); + self.write(desc + 14, next); + if i + 1 < chain.len() { + id = next; + } + } + self.free_head = self.next[usize::from(id)]; + self.num_free -= len; + self.in_flight[usize::from(head)] = len; + let slot = self.avail + 4 + 2 * usize::from(self.next_avail % self.size); + self.write(slot, head); + // The device must see the descriptors and slot before the index. + fence(Ordering::SeqCst); + self.next_avail = self.next_avail.wrapping_add(1); + self.write(self.avail + 2, self.next_avail); + Some(head) + } + + /// The next chain the device returned: its head, and how many bytes the + /// device says it wrote (untrusted; clamp before use). Entries for + /// chains that are not in flight are skipped. + pub fn pop_used(&mut self) -> Option<(u16, u32)> { + loop { + let used_idx: u16 = self.read(self.used + 2); + if used_idx == self.last_used { + return None; + } + fence(Ordering::Acquire); + let elem = self.used + 4 + 8 * usize::from(self.last_used % self.size); + let (id, len): (u32, u32) = (self.read(elem), self.read(elem + 4)); + self.last_used = self.last_used.wrapping_add(1); + let Some(head) = u16::try_from(id).ok().filter(|&id| id < self.size) else { + continue; + }; + let chain_len = core::mem::take(&mut self.in_flight[usize::from(head)]); + if chain_len == 0 { + continue; + } + let mut tail = head; + for _ in 1..chain_len { + tail = self.next[usize::from(tail)]; + } + self.next[usize::from(tail)] = self.free_head; + self.free_head = head; + self.num_free += chain_len; + return Some((head, len)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dma::Mmio; + + /// Heap memory, with physical = virtual addresses. + struct HeapHal; + + fn layout(pages: usize) -> core::alloc::Layout { + core::alloc::Layout::from_size_align(pages * crate::dma::PAGE_SIZE, 4096).unwrap() + } + + // Safety: heap memory is contiguous and exclusively owned. + unsafe impl Hal for HeapHal { + fn dma_alloc(&self, pages: usize) -> Option<(core::ptr::NonNull, u64)> { + // Safety: non-zero size. + let va = core::ptr::NonNull::new(unsafe { alloc::alloc::alloc(layout(pages)) })?; + Some((va, va.as_ptr() as u64)) + } + + unsafe fn dma_dealloc(&self, va: core::ptr::NonNull, pages: usize) { + // Safety: from `dma_alloc` with the same layout. + unsafe { alloc::alloc::dealloc(va.as_ptr(), layout(pages)) }; + } + + unsafe fn map_mmio(&self, _pa: u64, _len: usize) -> Option { + None + } + } + + fn queue(size: u16) -> VirtQueue { + VirtQueue::new(&HeapHal, 0, size, 0).unwrap() + } + + /// Plays the device: returns `id` with `len` in the used ring. + fn device_returns(queue: &VirtQueue, slot: u16, id: u32, len: u32) { + let elem = queue.used + 4 + 8 * usize::from(slot % queue.size); + queue.write(elem, id); + queue.write(elem + 4, len); + queue.write(queue.used + 2, slot.wrapping_add(1)); + } + + fn buffer(device_writes: bool) -> Buffer { + Buffer { + pa: 0x1000, + len: 8, + device_writes, + } + } + + #[test] + fn chains_are_published_and_freed() { + let mut q = queue(4); + // Safety: the device never touches the (fake) buffers. + let head = unsafe { q.add(&[buffer(false), buffer(true)]) }.unwrap(); + assert_eq!(q.num_free, 2); + let avail_idx: u16 = q.read(q.avail + 2); + assert_eq!(avail_idx, 1); + let flags: u16 = q.read(usize::from(head) * DESC_SIZE + 12); + assert_eq!(flags, DESC_F_NEXT); + assert_eq!(q.pop_used(), None); + device_returns(&q, 0, u32::from(head), 5); + assert_eq!(q.pop_used(), Some((head, 5))); + assert_eq!(q.num_free, 4); + } + + #[test] + fn bogus_and_replayed_used_entries_are_skipped() { + let mut q = queue(4); + // Safety: as above. + let head = unsafe { q.add(&[buffer(true)]) }.unwrap(); + device_returns(&q, 0, 1000, 1); // out of range + assert_eq!(q.pop_used(), None); + device_returns(&q, 1, u32::from(head) + 1, 1); // not in flight + assert_eq!(q.pop_used(), None); + device_returns(&q, 2, u32::from(head), 1); + assert_eq!(q.pop_used(), Some((head, 1))); + device_returns(&q, 3, u32::from(head), 1); // replayed + assert_eq!(q.pop_used(), None); + assert_eq!(q.num_free, 4); + // The free list is intact: the whole queue can be used again. + // Safety: as above. + assert!(unsafe { q.add(&[buffer(true); 4]) }.is_some()); + } +} diff --git a/litebox_platform_vm_kernel/Cargo.toml b/litebox_platform_vm_kernel/Cargo.toml index dc9e9fce6a..555d12ac62 100644 --- a/litebox_platform_vm_kernel/Cargo.toml +++ b/litebox_platform_vm_kernel/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] litebox = { path = "../litebox/", version = "0.1.0" } litebox_common_linux = { path = "../litebox_common_linux/", version = "0.1.0" } +litebox_hal = { path = "../litebox_hal", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } spin = { version = "0.10.0", default-features = false, features = [ diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.S b/litebox_platform_vm_kernel/src/arch/x86/interrupts.S index f68070dbda..2f861e4cc6 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.S +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.S @@ -143,26 +143,57 @@ isr_ignore: iretq /* - * IDT entries require an 8-byte stride; CALL pushes stub + 5 for vector lookup. - * Kernel-fatal even from user mode; GS may still belong to user code. + * External interrupts (vectors 32..=255), handled by the registered handler + * or fatal. IDT entries require an 8-byte stride; CALL pushes stub + 5 for + * vector lookup. + * + * They arrive from user mode, or from the kernel only while it halts with + * IF=1. Entry does not depend on GS (which may still be the user's): the + * handler must not touch per-CPU data, so no SWAPGS. The kernel runs with + * SSE disabled, so only the caller-saved GPRs need saving. */ .balign 8 -.global isr_unexpected_stubs -isr_unexpected_stubs: +.global isr_external_stubs +isr_external_stubs: .rept 224 .balign 8 - call isr_unexpected + call isr_external .endr -isr_unexpected: +isr_external: cld - pop rdi /* stub return address */ - lea rax, [rip + isr_unexpected_stubs + 5] + clac /* SMAP stays enforced; IRETQ restores AC */ + /* [rsp] stub return address, [rsp+8] RIP, [rsp+16] CS */ + push rax + push rcx + push rdx + push rsi + push rdi + push r8 + push r9 + push r10 + push r11 + push rbp + mov rbp, rsp + mov rdi, [rbp + 80] /* stub return address */ + lea rax, [rip + isr_external_stubs + 5] sub rdi, rax shr rdi, 3 add rdi, 32 /* vector */ - mov rsi, [rsp] /* interrupted RIP */ - mov rdx, [rsp + 8] /* interrupted CS */ + mov rsi, [rbp + 88] /* interrupted RIP */ + mov rdx, [rbp + 96] /* interrupted CS */ and rsp, -16 - call unexpected_interrupt_handler_impl - ud2 + call external_interrupt_handler_impl + mov rsp, rbp + pop rbp + pop r11 + pop r10 + pop r9 + pop r8 + pop rdi + pop rsi + pop rdx + pop rcx + pop rax + add rsp, 8 /* stub return address */ + iretq diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs index e3835e3a69..9a67d13282 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs @@ -5,6 +5,7 @@ //! NMI entry cannot assume kernel GS or a completed syscall stack switch. //! CR4.MCE remains off; machine checks shut down the CPU. +use crate::InterruptHandler; use litebox_common_linux::PtRegs; use spin::Once; use x86_64::{VirtAddr, structures::idt::InterruptDescriptorTable}; @@ -27,67 +28,83 @@ unsafe extern "C" { fn isr_alignment_check(); fn isr_simd_floating_point(); fn isr_ignore(); - fn isr_unexpected_stubs(); + /// One stub per vector from 32, each entering `isr_external`. + fn isr_external_stubs(); } -/// Stride of the per-vector stubs in `isr_unexpected_stubs`. +/// Stride of the per-vector stubs in `isr_external_stubs`. const UNEXPECTED_STUB_SIZE: usize = 8; const DOUBLE_FAULT_IST_INDEX: u16 = 0; const NMI_IST_INDEX: u16 = 1; +/// The IDT, and the state of the external interrupts it dispatches. +struct Interrupts { + idt: InterruptDescriptorTable, + /// See [`crate::set_interrupt_handler`]. + handler: Once<&'static InterruptHandler>, +} + +static INTERRUPTS: Once = Once::new(); + fn idt(ignored_vectors: &[u8]) -> &'static InterruptDescriptorTable { - static IDT_ONCE: Once = Once::new(); - IDT_ONCE.call_once(|| { - let mut idt = InterruptDescriptorTable::new(); - - // Safety: the stubs follow the interrupt calling convention. - unsafe { - idt.divide_error - .set_handler_addr(VirtAddr::from_ptr(isr_divide_error as *const ())); - idt.debug - .set_handler_addr(VirtAddr::from_ptr(isr_debug as *const ())); - idt.breakpoint - .set_handler_addr(VirtAddr::from_ptr(isr_breakpoint as *const ())); - idt.overflow - .set_handler_addr(VirtAddr::from_ptr(isr_overflow as *const ())); - idt.bound_range_exceeded - .set_handler_addr(VirtAddr::from_ptr(isr_bound_range_exceeded as *const ())); - idt.invalid_opcode - .set_handler_addr(VirtAddr::from_ptr(isr_invalid_opcode as *const ())); - idt.device_not_available - .set_handler_addr(VirtAddr::from_ptr(isr_device_not_available as *const ())); - for vector in 32..=u8::MAX { - let stub = isr_unexpected_stubs as *const () as usize - + usize::from(vector - 32) * UNEXPECTED_STUB_SIZE; - idt[vector].set_handler_addr(VirtAddr::new(stub as u64)); - } - idt.non_maskable_interrupt - .set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())) - .set_stack_index(NMI_IST_INDEX); - idt.double_fault - .set_handler_addr(VirtAddr::from_ptr(isr_double_fault as *const ())) - .set_stack_index(DOUBLE_FAULT_IST_INDEX); - idt.stack_segment_fault - .set_handler_addr(VirtAddr::from_ptr(isr_stack_segment_fault as *const ())); - idt.general_protection_fault - .set_handler_addr(VirtAddr::from_ptr( - isr_general_protection_fault as *const (), - )); - idt.page_fault - .set_handler_addr(VirtAddr::from_ptr(isr_page_fault as *const ())); - idt.x87_floating_point - .set_handler_addr(VirtAddr::from_ptr(isr_x87_floating_point as *const ())); - idt.alignment_check - .set_handler_addr(VirtAddr::from_ptr(isr_alignment_check as *const ())); - idt.simd_floating_point - .set_handler_addr(VirtAddr::from_ptr(isr_simd_floating_point as *const ())); - for &vector in ignored_vectors { - idt[vector].set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())); - } + &INTERRUPTS + .call_once(|| Interrupts { + idt: build_idt(ignored_vectors), + handler: Once::new(), + }) + .idt +} + +fn build_idt(ignored_vectors: &[u8]) -> InterruptDescriptorTable { + let mut idt = InterruptDescriptorTable::new(); + + // Safety: the stubs follow the interrupt calling convention. + unsafe { + idt.divide_error + .set_handler_addr(VirtAddr::from_ptr(isr_divide_error as *const ())); + idt.debug + .set_handler_addr(VirtAddr::from_ptr(isr_debug as *const ())); + idt.breakpoint + .set_handler_addr(VirtAddr::from_ptr(isr_breakpoint as *const ())); + idt.overflow + .set_handler_addr(VirtAddr::from_ptr(isr_overflow as *const ())); + idt.bound_range_exceeded + .set_handler_addr(VirtAddr::from_ptr(isr_bound_range_exceeded as *const ())); + idt.invalid_opcode + .set_handler_addr(VirtAddr::from_ptr(isr_invalid_opcode as *const ())); + idt.device_not_available + .set_handler_addr(VirtAddr::from_ptr(isr_device_not_available as *const ())); + for vector in 32..=u8::MAX { + let stub = isr_external_stubs as *const () as usize + + usize::from(vector - 32) * UNEXPECTED_STUB_SIZE; + idt[vector].set_handler_addr(VirtAddr::new(stub as u64)); } - idt - }) + idt.non_maskable_interrupt + .set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())) + .set_stack_index(NMI_IST_INDEX); + idt.double_fault + .set_handler_addr(VirtAddr::from_ptr(isr_double_fault as *const ())) + .set_stack_index(DOUBLE_FAULT_IST_INDEX); + idt.stack_segment_fault + .set_handler_addr(VirtAddr::from_ptr(isr_stack_segment_fault as *const ())); + idt.general_protection_fault + .set_handler_addr(VirtAddr::from_ptr( + isr_general_protection_fault as *const (), + )); + idt.page_fault + .set_handler_addr(VirtAddr::from_ptr(isr_page_fault as *const ())); + idt.x87_floating_point + .set_handler_addr(VirtAddr::from_ptr(isr_x87_floating_point as *const ())); + idt.alignment_check + .set_handler_addr(VirtAddr::from_ptr(isr_alignment_check as *const ())); + idt.simd_floating_point + .set_handler_addr(VirtAddr::from_ptr(isr_simd_floating_point as *const ())); + for &vector in ignored_vectors { + idt[vector].set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())); + } + } + idt } /// Requires [`super::gdt::init`]. The first call fixes `ignored_vectors`; @@ -104,8 +121,32 @@ pub fn init_idt(ignored_vectors: &[u8]) { idt(ignored_vectors).load(); } +/// # Panics +/// +/// Before [`init_idt`], or if a handler is already set. +pub(crate) fn set_external_interrupt_handler(handler: &'static InterruptHandler) { + let interrupts = INTERRUPTS.get().expect("the IDT is initialized at boot"); + let mut installed = false; + interrupts.handler.call_once(|| { + installed = true; + handler + }); + assert!(installed, "an interrupt handler is already set"); +} + +/// Runs with interrupts disabled and the interrupted context's GS (see +/// `isr_external`), on the interrupted stack from kernel mode or on the +/// `TSS.RSP0` stack from user mode. #[unsafe(no_mangle)] -extern "C" fn unexpected_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) -> ! { +extern "C" fn external_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) { + if let (Some(interrupts), Ok(vector)) = (INTERRUPTS.get(), u8::try_from(vector)) + && interrupts + .handler + .get() + .is_some_and(|handler| handler(vector)) + { + return; + } panic!("EXCEPTION: UNEXPECTED INTERRUPT (vector {vector:#x}, RIP {rip:#x}, CS {cs:#x})"); } diff --git a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs index 16fa733d62..8b7f445fed 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs @@ -436,6 +436,65 @@ impl X64PageTable<'_, M, ALIGN> { Ok(()) } + /// Gives the kernel PML4 slot holding `va` its P3 table now, as Linux + /// preallocates its vmalloc area's: task tables share the kernel's slots + /// as they are at creation, so they see later mappings in it. + /// + /// # Panics + /// + /// Panics if `va` is below the kernel slots or the frame allocation fails. + pub(crate) fn preallocate_kernel_slot(&self, va: VirtAddr) { + assert!(usize::from(va.p4_index()) >= KERNEL_PML4_START); + let mut inner = self.inner.lock(); + let entry = &mut inner.level_4_table_mut()[va.p4_index()]; + if entry.is_unused() { + let frame = PageTableAllocator::::allocate_frame() + .expect("Failed to allocate a kernel P3 table"); + entry.set_frame( + frame, + PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::ACCESSED + | PageTableFlags::DIRTY, + ); + } + } + + /// Maps device registers `frames` uncacheable at `start`, in the kernel + /// tables that task tables share. + /// + /// # Safety + /// + /// The frames must be device registers, not RAM, and `start..` kernel + /// address space in an existing kernel PML4 slot, used for nothing else. + pub(crate) unsafe fn map_kernel_mmio( + &self, + start: Page, + frames: PhysFrameRange, + ) -> Result<(), MapToError> { + let mut allocator = PageTableAllocator::::new(); + let table_flags = PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::ACCESSED + | PageTableFlags::DIRTY; + // PCD and PWT select UC with the reset PAT. + let flags = PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::NO_EXECUTE + | PageTableFlags::NO_CACHE + | PageTableFlags::WRITE_THROUGH; + let mut inner = self.inner.lock(); + for (page, frame) in Page::range(start, start + frames.count() as u64).zip(frames) { + // Safety: device registers (the caller's contract), mapped only here. + unsafe { + inner.map_to_with_table_flags(page, frame, flags, table_flags, &mut allocator) + }? + // A newly present entry needs no flush. + .ignore(); + } + Ok(()) + } + /// # Panics /// /// Panics if the frame allocation fails. diff --git a/litebox_platform_vm_kernel/src/clock.rs b/litebox_platform_vm_kernel/src/clock.rs index a607fc5194..1f1132ee3c 100644 --- a/litebox_platform_vm_kernel/src/clock.rs +++ b/litebox_platform_vm_kernel/src/clock.rs @@ -9,25 +9,8 @@ pub trait ClockSource: Sync { fn monotonic_nanos(&self) -> u64; } -/// Requires a single CPU and a constant TSC rate; neither is checked. -pub struct TscClock { - khz: u64, -} - -impl TscClock { - /// # Panics - /// - /// Panics if `khz` is zero. - pub fn new(khz: u64) -> Self { - assert!(khz != 0, "TSC frequency must not be zero"); - Self { khz } - } -} - -impl ClockSource for TscClock { +impl ClockSource for litebox_hal::clock::Clock { fn monotonic_nanos(&self) -> u64 { - // Safety: RDTSC has no side effects. - let tsc = u128::from(unsafe { core::arch::x86_64::_rdtsc() }); - u64::try_from(tsc * 1_000_000 / u128::from(self.khz)).unwrap_or(u64::MAX) + self.nanos() } } diff --git a/litebox_platform_vm_kernel/src/dma.rs b/litebox_platform_vm_kernel/src/dma.rs new file mode 100644 index 0000000000..982032db5b --- /dev/null +++ b/litebox_platform_vm_kernel/src/dma.rs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! [`litebox_hal::dma::Hal`]: DMA from the page allocator (RAM is +//! direct-mapped, so contiguous), and device memory mapped uncacheable in +//! [`IOREMAP`], each mapping followed by an unmapped guard page, as Linux's +//! `ioremap` does. + +use crate::mm::MemoryProvider; +use crate::{IOREMAP, VmKernel}; +use core::ptr::NonNull; +use litebox_hal::dma::{Hal, Mmio}; +use x86_64::structures::paging::{Page, PageSize, PhysFrame, Size4KiB}; +use x86_64::{PhysAddr, VirtAddr}; + +fn order(pages: usize) -> u32 { + pages.next_power_of_two().trailing_zeros() +} + +// Safety: page-allocator blocks are contiguous, exclusively owned until +// freed, and mapped writable at `PA + KERNEL_OFFSET`. Device memory is +// mapped uncacheable for the kernel's lifetime, and RAM is refused. +unsafe impl Hal for VmKernel { + fn dma_alloc(&self, pages: usize) -> Option<(NonNull, u64)> { + let va = NonNull::new(Self::mem_allocate_pages(order(pages))?)?; + Some((va, Self::va_to_pa(VirtAddr::from_ptr(va.as_ptr())).as_u64())) + } + + unsafe fn dma_dealloc(&self, va: NonNull, pages: usize) { + // Safety: from `dma_alloc` with the same `pages`. + unsafe { Self::mem_free_pages(va.as_ptr(), order(pages)) }; + } + + unsafe fn map_mmio(&self, pa: u64, len: usize) -> Option { + let end = pa + .checked_add(u64::try_from(len).ok()?) + .filter(|_| len != 0)?; + let first = PhysFrame::::containing_address(PhysAddr::try_new(pa).ok()?); + // Not `PhysAddr::align_up`, which panics past the physical address + // width. + let last = PhysFrame::containing_address( + PhysAddr::try_new(end.checked_next_multiple_of(Size4KiB::SIZE)?).ok()?, + ); + if self + .ram_frame_ranges + .iter() + .any(|r| r.start < last && first < r.end) + { + return None; + } + let start = { + let mut next = self.ioremap_next.lock(); + let start = *next; + let after = start + .checked_add(last.start_address() - first.start_address())? + .checked_add(Size4KiB::SIZE)?; + if after > IOREMAP.end { + return None; + } + *next = after; + start + }; + let page = Page::from_start_address(VirtAddr::new(start)).ok()?; + // Safety: device registers (the caller's contract), not RAM (checked), + // in a part of `IOREMAP` only this mapping gets. + unsafe { + self.page_table_manager + .base_page_table + .map_kernel_mmio(page, PhysFrame::range(first, last)) + } + .ok()?; + let va = page.start_address() + (pa - first.start_address().as_u64()); + // Safety: mapped uncacheable above, for the kernel's lifetime. + Some(unsafe { Mmio::new(NonNull::new(va.as_mut_ptr())?, len) }) + } +} diff --git a/litebox_platform_vm_kernel/src/lib.rs b/litebox_platform_vm_kernel/src/lib.rs index 04e4dc88ce..281e074ad5 100644 --- a/litebox_platform_vm_kernel/src/lib.rs +++ b/litebox_platform_vm_kernel/src/lib.rs @@ -12,8 +12,15 @@ //! //! Assumptions: a single CPU, no scheduler, and not a confidential VM (no //! #VE/#VC/#HV). -//! There is no timer, so user code that never enters the kernel is never -//! preempted. +//! +//! Devices: [`VmKernel`] is the drivers' [`litebox_hal::dma::Hal`]. +//! +//! Interrupts: the kernel runs with interrupts disabled, except while it +//! halts ([`VmKernel::halt_until_interrupt`]); user code runs with them +//! enabled. External interrupts go to the handler the runner installs +//! ([`set_interrupt_handler`]), which returns to the interrupted code: there +//! is no preemption, so user code that never enters the kernel keeps the +//! CPU. #![cfg(target_arch = "x86_64")] #![no_std] @@ -67,6 +74,7 @@ extern crate alloc; mod arch; mod boot; pub mod clock; +mod dma; pub mod mm; mod per_cpu; pub mod providers; @@ -89,6 +97,26 @@ pub struct BootConfig<'a> { pub ignored_vectors: &'a [u8], } +/// See [`set_interrupt_handler`]. +pub type InterruptHandler = dyn Fn(u8) -> bool + Sync; + +/// Installs the handler for external interrupts (vectors 32 and up, except +/// [`BootConfig::ignored_vectors`]), which returns whether it handled the +/// vector; an unhandled one panics. Before this, every external interrupt +/// panics. +/// +/// The handler runs with interrupts disabled, on the interrupted context's GS +/// (user mode, or [`VmKernel::halt_until_interrupt`]): it must not touch +/// per-CPU data, block, or take locks; it should only end the interrupt and +/// leave the work to the kernel after the halt. +/// +/// # Panics +/// +/// On a second call. +pub fn set_interrupt_handler(handler: &'static InterruptHandler) { + arch::interrupts::set_external_interrupt_handler(handler); +} + /// Valid until unregistered; its representation is private to the platform. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct AddressSpaceId(usize); @@ -97,14 +125,21 @@ impl AddressSpaceId { pub const KERNEL: Self = Self(0); } -// Virtual address space: -// 0xFFFF_E200_0000_0000 .. kernel: all owned RAM at PA + KERNEL_OFFSET -// 0x0000_0000_0001_0000 .. 0x0000_7FFF_FFFF_F000 user +// Virtual address space, one 512 GiB PML4 slot per kernel region, as in +// Linux's Documentation/arch/x86/x86_64/mm.rst: +// 0x0000_0000_0001_0000 .. 0x0000_7FFF_FFFF_F000 user, per task +// 0xFFFF_E200_0000_0000 .. 0xFFFF_E280_0000_0000 direct map of all RAM, +// at PA + KERNEL_OFFSET +// 0xFFFF_E280_0000_0000 .. 0xFFFF_E300_0000_0000 ioremap: device memory /// `VA = PA + KERNEL_OFFSET` for kernel-owned RAM. Must be 512 GiB aligned /// (one PML4 slot). pub const KERNEL_OFFSET: u64 = 0xFFFF_E200_0000_0000; +/// Device memory mappings ([`litebox_hal::dma::Hal::map_mmio`]), the slot +/// after the direct map, which RAM must end below. +const IOREMAP: core::ops::Range = KERNEL_OFFSET + (512 << 30)..KERNEL_OFFSET + (1024 << 30); + /// Exclusive; the last page of the low canonical half is a guard page. const USER_ADDR_MAX: usize = 0x0000_7FFF_FFFF_F000; @@ -281,6 +316,8 @@ pub struct VmKernel { clock: &'static dyn clock::ClockSource, /// Guest RAM owned by this kernel (mapped at `PA + KERNEL_OFFSET`). ram_frame_ranges: alloc::vec::Vec>, + /// The next free address in [`IOREMAP`]. + ioremap_next: spin::Mutex, } /// Confines user pointers to the user range and opens SMAP around accesses. @@ -411,7 +448,14 @@ impl VmKernel { "kernel text and read-only data overlap" ); + assert!( + ram_frame_ranges + .iter() + .all(|r| KERNEL_OFFSET + r.end.start_address().as_u64() <= IOREMAP.start), + "guest RAM extends past the direct map" + ); let base_pt = mm::PageTable::new_top_level(); + base_pt.preallocate_kernel_slot(VirtAddr::new(IOREMAP.start)); for range in &ram_frame_ranges { if let Err(e) = base_pt.map_kernel_ram(*range, text, read_only) { panic!("failed to map guest RAM {range:?}: {e:?}"); @@ -430,9 +474,19 @@ impl VmKernel { page_table_manager: PageTableManager::new(base_pt), clock, ram_frame_ranges, + ioremap_next: spin::Mutex::new(IOREMAP.start), })) } + /// Halts until an interrupt is handled, with interrupts enabled only for + /// the halt. An interrupt that became pending while they were disabled + /// ends the halt at once, so check for work first and halt after: none + /// is missed. + pub fn halt_until_interrupt(&self) { + x86_64::instructions::interrupts::enable_and_hlt(); + x86_64::instructions::interrupts::disable(); + } + /// Only valid for addresses in the kernel's direct mapping. pub fn va_to_pa(va: VirtAddr) -> PhysAddr { ::va_to_pa(va) diff --git a/litebox_runner_optee_on_qemu/src/main.rs b/litebox_runner_optee_on_qemu/src/main.rs index 2adcfb2dce..d1c4436a53 100644 --- a/litebox_runner_optee_on_qemu/src/main.rs +++ b/litebox_runner_optee_on_qemu/src/main.rs @@ -18,7 +18,7 @@ mod tests; use core::panic::PanicInfo; use litebox_bootloader::Kernel; -use litebox_hal::{console, console_println, power}; +use litebox_hal::{power, serial, serial_println}; use litebox_platform_vm_kernel::providers::set_platform_root_key; litebox_bootloader::entry!(kernel_main, &SERIAL_LOGGER); @@ -28,7 +28,7 @@ fn kernel_main(kernel: Kernel) -> ! { tests::check_user_memory_protection(kernel.platform); optee::run(kernel.platform, kernel.boot_info); - console_println!("[litebox] ALL TESTS PASSED"); + serial_println!("[litebox] ALL TESTS PASSED"); power::exit(true) } @@ -45,7 +45,7 @@ impl log::Log for SerialLogger { } let mut buf: arrayvec::ArrayString<1024> = arrayvec::ArrayString::new(); let _ = litebox_util_log::format_record(&mut buf, record); - console::print_str(&buf); + serial::print_str(&buf); } fn flush(&self) {} @@ -69,7 +69,7 @@ static SERIAL_LOGGER: SerialLogger = SerialLogger; #[panic_handler] fn panic(info: &PanicInfo) -> ! { - console::panic_print(format_args!( + serial::panic_print(format_args!( "[litebox] PANIC: {info}\n[litebox] TESTS FAILED\n" )); power::exit(false) diff --git a/litebox_runner_optee_on_vm_userland/Cargo.toml b/litebox_runner_optee_on_vm_userland/Cargo.toml index df94def7d4..beb44fa899 100644 --- a/litebox_runner_optee_on_vm_userland/Cargo.toml +++ b/litebox_runner_optee_on_vm_userland/Cargo.toml @@ -6,6 +6,7 @@ description = "Runs an OP-TEE TA in ring 3 on top of the LiteBox VM kernel" [dependencies] litebox = { version = "0.1.0", path = "../litebox" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_common_linux = { version = "0.1.0", path = "../litebox_common_linux" } litebox_common_optee = { version = "0.1.0", path = "../litebox_common_optee" } litebox_broker_local_vm_userland = { version = "0.1.0", path = "../litebox_broker_local_vm_userland" } diff --git a/litebox_runner_optee_on_vm_userland/src/optee.rs b/litebox_runner_optee_on_vm_userland/src/optee.rs index 5f98bdbe6d..9b2d8fc972 100644 --- a/litebox_runner_optee_on_vm_userland/src/optee.rs +++ b/litebox_runner_optee_on_vm_userland/src/optee.rs @@ -15,6 +15,7 @@ use core::cell::RefCell; use core::ops::Range; use litebox::platform::RawConstPointer as _; use litebox::utils::TruncateExt as _; +use litebox_broker_protocol::fs::{FileAccessMode, FileMode, FileOpenFlags}; use litebox_common_linux::PtRegs; use litebox_common_optee::envelope::{ LDELF_IMAGE, OPEN_SESSION_META_PARAMS, TA_IMAGE, parse_open_session_meta, rmem_buffer, @@ -132,14 +133,28 @@ pub fn serve(info: &StartupInfo) -> ! { let platform: &'static Platform = Box::leak(Box::new(VmUserland::new(info))); let session_manager: &'static SessionManager = Box::leak(Box::new(SessionManager::new())); - let local = litebox_broker_local_vm_userland::connect(info) + let (local, mut notifications) = litebox_broker_local_vm_userland::connect(info) .unwrap_or_else(|e| panic!("broker association: {e:?}")); kcall::restrict(CallSet::ALL.without(CallId::BrokerHandshake), ProtSet::ALL) .unwrap_or_else(|status| panic!("lockdown: {status:?}")); let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let shim = - litebox_shim_optee::OpteeShimBuilder::new_with_litebox(platform, session_manager, litebox) - .build(); + let mut builder = + litebox_shim_optee::OpteeShimBuilder::new_with_litebox(platform, session_manager, litebox); + // TA and ldelf output goes to the broker's standard output. + match builder.litebox().open_file( + &litebox::fs::Context::new(), + "/dev/stdout", + FileAccessMode::WriteOnly, + FileOpenFlags::NONE, + FileMode::empty(), + ) { + Ok(stdout) => builder = builder.with_log_file(stdout), + Err(error) => log::warn!("TA output stays in the log: /dev/stdout: {error:?}"), + } + if cfg!(debug_assertions) { + check_stdin(builder.litebox(), &mut notifications, info.tsc_khz); + } + let shim = builder.build(); // Safety: mapped read-only for the process's lifetime; nothing writes // them. @@ -171,7 +186,7 @@ pub fn serve(info: &StartupInfo) -> ! { // Requires upcalls, registered by `ready`. check_guest_memory_access(); check_lockdown_validation(); - litebox_broker_local_vm_userland::self_check(); + litebox_broker_local_vm_userland::self_check(¬ifications, info.tsc_khz); } loop { let reply = runner.serve(request); @@ -729,6 +744,64 @@ fn check_guest_memory_access() { assert_eq!(read(), None, "read from unmapped guest memory"); } +/// Self-check (debug builds): standard input through the broker, which TAs +/// never read. Waits briefly for input (on broker notifications), and echoes +/// what arrived to standard output. +fn check_stdin( + litebox: &litebox::LiteBox, + notifications: &mut litebox_broker_local_vm_userland::Notifications, + tsc_khz: u64, +) { + use litebox::fs::errors::ReadError; + const WAIT_MS: u64 = 20; + + let context = litebox::fs::Context::new(); + let open = |path, access| { + litebox + .open_file( + &context, + path, + access, + FileOpenFlags::NONBLOCKING, + FileMode::empty(), + ) + .unwrap_or_else(|error| panic!("opening {path}: {error:?}")) + }; + let stdin = open("/dev/stdin", FileAccessMode::ReadOnly); + // Safety: RDTSC has no side effects; the kernel allows it in ring 3. + let deadline = unsafe { core::arch::x86_64::_rdtsc() } + WAIT_MS * tsc_khz; + let mut input = [0u8; 256]; + let len = loop { + match litebox.read_file(&stdin, &mut input, None, None) { + Ok(len) => break len, + Err(ReadError::WouldBlock) => { + let dispatch = litebox.broker_notification_dispatcher(); + match notifications.receive(deadline, dispatch) { + Ok(0) => break 0, + Ok(_) => {} + Err(error) => panic!("waiting for standard input: {error}"), + } + } + Err(error) => panic!("reading standard input: {error:?}"), + } + }; + litebox.close_file(&stdin).expect("closing standard input"); + if len == 0 { + log::debug!("stdin check: no input within {WAIT_MS} ms"); + return; + } + let stdout = open("/dev/stdout", FileAccessMode::WriteOnly); + for bytes in [&b"[stdin] "[..], &input[..len]] { + litebox + .write_file(&stdout, bytes, None, None) + .expect("writing standard output"); + } + litebox + .close_file(&stdout) + .expect("closing standard output"); + log::info!("stdin check: echoed {len} bytes"); +} + /// Self-check (debug builds): unknown lockdown bits are rejected. fn check_lockdown_validation() { use litebox_common_vm_abi::{RestrictRequest, Status}; diff --git a/litebox_runner_vm_kernel/Cargo.toml b/litebox_runner_vm_kernel/Cargo.toml index dbe6f2ea8b..336723d896 100644 --- a/litebox_runner_vm_kernel/Cargo.toml +++ b/litebox_runner_vm_kernel/Cargo.toml @@ -16,6 +16,7 @@ litebox_platform_vm_kernel = { version = "0.1.0", path = "../litebox_platform_vm litebox_shim_vm_kernel = { version = "0.1.0", path = "../litebox_shim_vm_kernel" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } log = { version = "0.4", default-features = false } +spin = { version = "0.10.0", default-features = false, features = ["spin_mutex"] } serde = { version = "1.0.142", default-features = false, features = ["derive", "alloc"] } serde_json = { version = "1.0.142", default-features = false, features = ["alloc"] } tar-no-std = { version = "0.3.3", default-features = false, features = ["alloc"] } diff --git a/litebox_runner_vm_kernel/src/devices.rs b/litebox_runner_vm_kernel/src/devices.rs new file mode 100644 index 0000000000..475f4aa63b --- /dev/null +++ b/litebox_runner_vm_kernel/src/devices.rs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Devices the kernel drives, from `litebox_hal`, for the broker +//! ([`litebox_broker_vm_kernel::Config`]): +//! +//! - [`Console`]: the VM's console device, else output to the serial port +//! and no input. +//! - [`Events`]: the devices' interrupt, and the local APIC's timer for +//! deadlines. +//! +//! The interrupt handler only ends the interrupt; the broker looks at the +//! devices when it next runs. + +use alloc::boxed::Box; +use core::time::Duration; +use litebox_broker_vm_kernel::Events; +use litebox_broker_vm_kernel::stdio::Console; +use litebox_hal::clock::{self, Clock}; +use litebox_hal::devices::ConsoleDevice; +use litebox_hal::interrupt::LocalApic; +use litebox_platform_vm_kernel::VmKernel; +use spin::Mutex; + +/// How long a console write may wait for the device to take output before +/// the rest is dropped. +const WRITE_TIMEOUT: Duration = Duration::from_millis(100); + +const TIMER_VECTOR: u8 = 0x30; +/// Shared by every device: the broker looks at all of them when it runs. +const DEVICE_VECTOR: u8 = 0x31; + +pub struct Devices { + platform: &'static VmKernel, + apic: LocalApic, + clock: Clock, + console: Mutex, + /// Whether the console interrupts; else it is polled. + console_interrupts: bool, +} + +impl Devices { + /// Brings up the devices and enables their interrupts. Call once. + pub fn init(platform: &'static VmKernel, clock: Clock) -> &'static Self { + let apic = LocalApic::init(platform, TIMER_VECTOR, clock); + let console = match ConsoleDevice::probe(platform, apic.target(DEVICE_VECTOR)) { + Ok(Some(device)) => device, + Ok(None) => ConsoleDevice::serial(), + Err(error) => { + litebox_util_log::warn!(error:% = error; "console device unusable"); + ConsoleDevice::serial() + } + }; + litebox_util_log::info!(device:% = console; "standard streams"); + let devices: &'static Self = Box::leak(Box::new(Self { + platform, + apic, + clock, + console_interrupts: console.interrupts(), + console: Mutex::new(console), + })); + litebox_platform_vm_kernel::set_interrupt_handler(Box::leak(Box::new(|vector| { + devices.on_interrupt(vector) + }))); + devices + } + + /// See `litebox_platform_vm_kernel::set_interrupt_handler`: no locks, no + /// per-CPU data. + fn on_interrupt(&self, vector: u8) -> bool { + if !matches!(vector, TIMER_VECTOR | DEVICE_VECTOR) { + return false; + } + self.apic.end_of_interrupt(); + true + } + + /// Halts until an interrupt, or until `deadline` (a [`clock::now`] value). + fn halt(&self, deadline: Option) { + if let Some(deadline) = deadline { + self.apic.arm_timer(deadline); + } + self.platform.halt_until_interrupt(); + if deadline.is_some() { + self.apic.disarm_timer(); + } + } +} + +impl Events for Devices { + fn wait(&self, deadline: Option) -> bool { + if deadline.is_none() && !self.console_interrupts { + return false; + } + self.halt(deadline); + true + } +} + +/// A console that receives nothing is at the end of its input. +impl Console for Devices { + fn read(&self, output: &mut [u8]) -> Option { + let mut console = self.console.lock(); + console.receives().then(|| console.read(output)) + } + + fn has_input(&self) -> bool { + let mut console = self.console.lock(); + !console.receives() || console.has_input() + } + + /// Waits (bounded) for the device to take all of `input`. + fn write(&self, input: &[u8]) -> usize { + let deadline = self.clock.deadline(WRITE_TIMEOUT); + let mut written = 0; + loop { + written += self.console.lock().write(&input[written..]); + if written == input.len() || clock::now() >= deadline { + return written; + } + // Until the device takes more output. + self.halt(Some(deadline)); + } + } +} diff --git a/litebox_runner_vm_kernel/src/main.rs b/litebox_runner_vm_kernel/src/main.rs index 4b46a44556..75a2108261 100644 --- a/litebox_runner_vm_kernel/src/main.rs +++ b/litebox_runner_vm_kernel/src/main.rs @@ -6,7 +6,8 @@ //! `litebox_runner_optee_on_vm_userland` processes, one per TA instance //! ([`service::optee`]), //! driven by a test client ([`client::script`]). QEMU exits with 33 if all -//! tests pass, 65 otherwise. +//! tests pass, 65 otherwise. Runner processes' standard streams are the +//! VM's console device, if there is one ([`devices`]). #![cfg(target_arch = "x86_64")] #![no_std] @@ -16,12 +17,13 @@ extern crate alloc; mod client; +mod devices; mod payload; mod service; use core::panic::PanicInfo; use litebox_bootloader::Kernel; -use litebox_hal::{console, console_println, power}; +use litebox_hal::{power, serial, serial_println}; use litebox_platform_vm_kernel::providers::set_platform_root_key; litebox_bootloader::entry!(kernel_main, &SerialLogger); @@ -30,9 +32,14 @@ fn kernel_main(kernel: Kernel) -> ! { set_platform_root_key(&litebox_hal::prk::development()); let payload = payload::Payload::read(kernel.boot_info).unwrap_or_else(|e| panic!("payload: {e}")); - let mut service = service::optee::Optee::new(kernel.platform, kernel.tsc_khz, &payload); + let devices = devices::Devices::init(kernel.platform, kernel.clock); + let broker = litebox_broker_vm_kernel::Broker::new(litebox_broker_vm_kernel::Config { + console: devices, + events: devices, + }); + let mut service = service::optee::Optee::new(kernel.platform, kernel.clock, &payload, broker); client::script::run(&payload, &mut service); - console_println!("[litebox] ALL TESTS PASSED"); + serial_println!("[litebox] ALL TESTS PASSED"); power::exit(true) } @@ -49,7 +56,7 @@ impl log::Log for SerialLogger { } let mut buf: arrayvec::ArrayString<1024> = arrayvec::ArrayString::new(); let _ = litebox_util_log::format_record(&mut buf, record); - console::print_str(&buf); + serial::print_str(&buf); } fn flush(&self) {} @@ -71,7 +78,7 @@ fn is_spurious_tar_error(record: &log::Record) -> bool { #[panic_handler] fn panic(info: &PanicInfo) -> ! { - console::panic_print(format_args!( + serial::panic_print(format_args!( "[litebox] PANIC: {info}\n[litebox] TESTS FAILED\n" )); power::exit(false) diff --git a/litebox_runner_vm_kernel/src/service/optee.rs b/litebox_runner_vm_kernel/src/service/optee.rs index c3dd7740bd..8a33f6c6df 100644 --- a/litebox_runner_vm_kernel/src/service/optee.rs +++ b/litebox_runner_vm_kernel/src/service/optee.rs @@ -13,6 +13,7 @@ use alloc::collections::BTreeMap; use litebox_common_optee::TeeUuid; use litebox_common_optee::envelope::{LDELF_IMAGE, TA_IMAGE}; use litebox_common_vm_abi::IDENTITY_LEN; +use litebox_hal::clock::Clock; use litebox_platform_vm_kernel::VmKernel; use litebox_shim_vm_kernel::optee::Completion; use litebox_shim_vm_kernel::optee::Invocation; @@ -36,7 +37,12 @@ impl Optee { /// # Panics /// /// On a missing or malformed payload file. - pub fn new(platform: &'static VmKernel, tsc_khz: u64, payload: &Payload) -> Self { + pub fn new( + platform: &'static VmKernel, + clock: Clock, + payload: &Payload, + broker: litebox_broker_vm_kernel::Broker, + ) -> Self { let file = |name| { payload .file(name) @@ -66,7 +72,6 @@ impl Optee { policies.push((uuid, policy)); } assert!(!tas.is_empty(), "payload has no tas/.elf"); - let broker = litebox_broker_vm_kernel::Broker::new(); let spawn: Spawn = Box::new(move |uuid: &TeeUuid| { let Some(ta) = tas.get(uuid) else { litebox_util_log::error!(uuid:? = uuid; "no such TA"); @@ -84,7 +89,7 @@ impl Optee { runner, images: &images, identity, - tsc_khz, + tsc_khz: clock.khz(), }, ) .map_err(|error| { diff --git a/litebox_shim_optee/src/lib.rs b/litebox_shim_optee/src/lib.rs index bfc98591be..da110e4953 100644 --- a/litebox_shim_optee/src/lib.rs +++ b/litebox_shim_optee/src/lib.rs @@ -185,6 +185,7 @@ pub struct OpteeShimBuilder { session_manager: &'static session::SessionManager, litebox: LiteBox, ta_signing_cert: &'static [u8], + log_file: Option, } impl OpteeShimBuilder { @@ -209,6 +210,7 @@ impl OpteeShimBuilder { session_manager, litebox, ta_signing_cert: &[], + log_file: None, } } @@ -219,6 +221,15 @@ impl OpteeShimBuilder { self } + /// Where TA and ldelf messages (`utee_log`) go, as written, instead of + /// the log: a file of [`Self::litebox`], such as the broker's + /// `/dev/stdout`. Messages the file does not take are logged. + #[must_use] + pub fn with_log_file(mut self, log_file: litebox::fs::FileFd) -> Self { + self.log_file = Some(log_file); + self + } + /// Returns the litebox object for the shim. pub fn litebox(&self) -> &LiteBox { &self.litebox @@ -236,6 +247,7 @@ impl OpteeShimBuilder { dynamic_ta_uuid_map: TaUuidMap::new(), ta_signing_cert: self.ta_signing_cert, pta_busy: spin::mutex::SpinMutex::new(HashSet::new()), + log_file: self.log_file, page_table_keepalive: None, }); OpteeShim(global) @@ -269,6 +281,8 @@ struct GlobalState { /// blocking/queuing the caller until the PTA is free. We currently reject /// instead of serialize; revisit if a PTA needs true serialization. pta_busy: spin::mutex::SpinMutex>, + /// See [`OpteeShimBuilder::with_log_file`]. + log_file: Option, /// Keeps the TA page table alive; declared last to drop after all other shim state. // TODO: Replace type erasure with a typed platform page-table handle. page_table_keepalive: Option>, diff --git a/litebox_shim_optee/src/syscalls/tee.rs b/litebox_shim_optee/src/syscalls/tee.rs index 363c27e1f5..8f2910795a 100644 --- a/litebox_shim_optee/src/syscalls/tee.rs +++ b/litebox_shim_optee/src/syscalls/tee.rs @@ -58,12 +58,26 @@ impl Task { litebox_common_optee::TeeResult::TargetDead as usize } - /// A system call to print out a message. - #[expect( - clippy::unused_self, - reason = "self was used by the old platform-threaded logging API" - )] + /// A system call to print out a message: to the log file, as is (OP-TEE + /// traces end their lines, and `printf` may log fragments), else to the + /// log. A message the file takes in part is not logged again. pub fn sys_log(&self, buf: &[u8]) -> Result<(), TeeResult> { + if let Some(file) = &self.global.log_file { + let mut written = 0; + while written < buf.len() { + match self + .global + .litebox + .write_file(file, &buf[written..], None, None) + { + Ok(n) if n != 0 => written += n, + _ => break, + } + } + if written != 0 || buf.is_empty() { + return Ok(()); + } + } let msg = core::str::from_utf8(buf).map_err(|_| TeeResult::BadFormat)?; litebox_util_log::info!(msg:% = msg; "sys_log"); Ok(())