From 0963d46a891b14015ed396945c7e73ad8df8907e Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Fri, 9 Oct 2026 20:14:05 +0000 Subject: [PATCH 1/7] add a virtio console for broker-based stdio --- Cargo.lock | 7 + dev_tools/qemu_test_lib.sh | 11 +- dev_tools/run_optee_on_vm_userland.sh | 36 ++- litebox_broker_host/src/readiness.rs | 40 +++ litebox_broker_local_vm_userland/src/lib.rs | 149 +++++++-- litebox_broker_vm_kernel/Cargo.toml | 1 + litebox_broker_vm_kernel/src/lib.rs | 132 ++++---- litebox_broker_vm_kernel/src/providers.rs | 30 +- litebox_broker_vm_kernel/src/readiness.rs | 115 +++++++ litebox_common_vm_abi/src/lib.rs | 10 +- litebox_hal/src/console.rs | 13 +- litebox_hal/src/dma.rs | 167 ++++++++++ litebox_hal/src/interrupt.rs | 72 ++++- litebox_hal/src/lib.rs | 9 + litebox_hal/src/pci.rs | 211 ++++++++++++ litebox_hal/src/timer.rs | 39 +++ litebox_hal/src/virtio/console.rs | 231 ++++++++++++++ litebox_hal/src/virtio/mod.rs | 68 ++++ litebox_hal/src/virtio/pci.rs | 302 ++++++++++++++++++ litebox_hal/src/virtio/queue.rs | 282 ++++++++++++++++ litebox_platform_vm_kernel/Cargo.toml | 1 + .../src/arch/x86/interrupts.S | 55 +++- .../src/arch/x86/interrupts.rs | 159 ++++++--- .../src/arch/x86/mm/paging.rs | 39 +++ litebox_platform_vm_kernel/src/dma.rs | 80 +++++ litebox_platform_vm_kernel/src/lib.rs | 60 +++- .../Cargo.toml | 1 + .../src/optee.rs | 83 ++++- litebox_runner_vm_kernel/Cargo.toml | 3 + litebox_runner_vm_kernel/src/devices.rs | 238 ++++++++++++++ litebox_runner_vm_kernel/src/main.rs | 11 +- litebox_runner_vm_kernel/src/service/optee.rs | 8 +- litebox_shim_optee/Cargo.toml | 1 + litebox_shim_optee/src/lib.rs | 14 + litebox_shim_optee/src/syscalls/tee.rs | 24 +- 35 files changed, 2511 insertions(+), 191 deletions(-) create mode 100644 litebox_broker_vm_kernel/src/readiness.rs create mode 100644 litebox_hal/src/dma.rs create mode 100644 litebox_hal/src/pci.rs create mode 100644 litebox_hal/src/timer.rs create mode 100644 litebox_hal/src/virtio/console.rs create mode 100644 litebox_hal/src/virtio/mod.rs create mode 100644 litebox_hal/src/virtio/pci.rs create mode 100644 litebox_hal/src/virtio/queue.rs create mode 100644 litebox_platform_vm_kernel/src/dma.rs create mode 100644 litebox_runner_vm_kernel/src/devices.rs diff --git a/Cargo.lock b/Cargo.lock index d122275159..e2f16a00d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,6 +1631,7 @@ dependencies = [ "litebox_hal", "litebox_platform_vm_kernel", "log", + "spin 0.10.0", ] [[package]] @@ -1846,6 +1847,7 @@ dependencies = [ "hashbrown", "litebox", "litebox_common_linux", + "litebox_hal", "litebox_platform", "litebox_util_log", "spin 0.10.0", @@ -2032,6 +2034,7 @@ dependencies = [ "aes", "litebox", "litebox_broker_local_vm_userland", + "litebox_broker_protocol", "litebox_common_linux", "litebox_common_optee", "litebox_common_vm_abi", @@ -2064,6 +2067,8 @@ dependencies = [ "arrayvec", "base64", "litebox_bootloader", + "litebox_broker_core", + "litebox_broker_protocol", "litebox_broker_vm_kernel", "litebox_common_optee", "litebox_common_vm_abi", @@ -2074,6 +2079,7 @@ dependencies = [ "log", "serde", "serde_json", + "spin 0.10.0", "tar-no-std", "zerocopy", ] @@ -2195,6 +2201,7 @@ dependencies = [ "hashbrown", "hmac", "litebox", + "litebox_broker_protocol", "litebox_common_linux", "litebox_common_optee", "litebox_platform", diff --git a/dev_tools/qemu_test_lib.sh b/dev_tools/qemu_test_lib.sh index a4f52d8d4a..b63a4cda30 100644 --- a/dev_tools/qemu_test_lib.sh +++ b/dev_tools/qemu_test_lib.sh @@ -52,10 +52,13 @@ pass=0 fail=0 failed=() -# run_vm NAME KERNEL PAYLOAD: boots KERNEL with PAYLOAD as its initrd and -# records the outcome; with $verbose set, prints passing logs too. +# run_vm NAME KERNEL PAYLOAD [QEMU_ARG]...: boots KERNEL with PAYLOAD as its +# initrd and records the outcome; with $verbose set, prints passing logs too. +# If $check names a function, a guest pass also needs `$check NAME` to +# succeed; it prints why not otherwise. run_vm() { local name=$1 kernel=$2 payload=$3 log="$WORK/$1.log" status why + shift 3 set +e # shellcheck disable=SC2086 # QEMU is a command line timeout "$TIMEOUT" $QEMU \ @@ -64,16 +67,18 @@ run_vm() { -kernel "$kernel" -initrd "$payload" -append "litebox.log=${LITEBOX_LOG:-info}" \ -serial stdio -display none -no-reboot \ -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ + "$@" \ "$log" 2>&1 status=$? set -e # isa-debug-exit: QEMU exits with (value << 1) | 1; 33 = pass. - if [[ $status -eq 33 ]]; then + if [[ $status -eq 33 ]] && { [[ -z ${check:-} ]] || why=$($check "$name"); }; then echo "PASS $name" pass=$((pass + 1)) [[ ${verbose:-0} -eq 0 ]] || cat "$log" else case $status in + 33) ;; 65) why="guest reported failure" ;; 124) why="timed out after ${TIMEOUT}s" ;; *) why="exit status $status" ;; diff --git a/dev_tools/run_optee_on_vm_userland.sh b/dev_tools/run_optee_on_vm_userland.sh index 60279d50e9..2dd7ce13f9 100755 --- a/dev_tools/run_optee_on_vm_userland.sh +++ b/dev_tools/run_optee_on_vm_userland.sh @@ -14,13 +14,17 @@ # open_session commands name the TAs it needs in their "ta" fields, or a TA # with a *-cmds.json of its own. # +# Runner processes' standard streams are a virtio console, backed by files: +# TA output must reach it, and in debug builds, whose runners echo standard +# input, so must the line fed to it. +# # Usage: dev_tools/run_optee_on_vm_userland.sh [-t ]... [-u] [-r] [-v] # -t test to run, e.g. ta-death or hello-ta; append @default to a TA to # omit its cmds.json (repeatable; default: every scenario, plus # hello-ta@default for the default-commands path) # -u use unmodified ldelf and TAs; the kernel reflects their syscalls # -r build and run the release kernel -# -v print the full guest log for passing runs too +# -v print the full guest log and console output for passing runs too # # Environment: # QEMU QEMU binary/command (default: qemu-system-x86_64) @@ -93,6 +97,33 @@ prepare() { fi } +STDIN_LINE="hello from the host" + +# The virtio console's files for test $1. +console_args() { + printf '%s\n' "$STDIN_LINE" >"$WORK/$1.stdin" + : >"$WORK/$1.stdout" + echo -device virtio-serial-pci,id=vs0,disable-legacy=on \ + -chardev "file,id=con0,path=$WORK/$1.stdout,input-path=$WORK/$1.stdin" \ + -device virtconsole,chardev=con0,bus=vs0.0 +} + +# Run check: TA output, and the echoed input, on the console. +check_console() { + local out="$WORK/$1.stdout" + if [[ ${verbose:-0} -ne 0 ]]; then + echo "--- console ($1) ---" >&2 + cat "$out" >&2 + echo "---" >&2 + fi + grep -q "/TA:" "$out" || { echo "no TA output on the console"; return 1; } + if [[ $profile == debug ]] && ! grep -qF "[stdin] $STDIN_LINE" "$out"; then + echo "standard input was not echoed on the console" + return 1 + fi +} +check=check_console + for ta in "${tas[@]}"; do name=${ta%@default} if [[ -f "$VM_TESTS_DIR/$name-cmds.json" ]]; then @@ -118,6 +149,7 @@ for ta in "${tas[@]}"; do files+=(cmds.json) fi tar --format=ustar -cf "$WORK/$ta.tar" -C "$WORK/$ta" "${files[@]}" - run_vm "$ta" "$KERNEL" "$WORK/$ta.tar" + # shellcheck disable=SC2046 # console_args prints separate arguments + run_vm "$ta" "$KERNEL" "$WORK/$ta.tar" $(console_args "$ta") done summary diff --git a/litebox_broker_host/src/readiness.rs b/litebox_broker_host/src/readiness.rs index c928dabddf..ad2736b293 100644 --- a/litebox_broker_host/src/readiness.rs +++ b/litebox_broker_host/src/readiness.rs @@ -380,6 +380,24 @@ pub fn publish_readiness( } } +/// Publishes every pending readiness update, then returns; for deployments +/// without a publisher thread, which call this whenever updates may be +/// pending. +/// +/// The channel must not block: a send that cannot proceed (e.g., a full +/// notification ring) fails, which leaves its update publishable for the next +/// call and returns the error. +pub fn publish_pending_readiness( + publisher: &ReadinessPublisher, + channel: &mut Channel, +) -> Result<(), Channel::Error> { + while let Some(pending) = publisher.take_pending() { + channel.send_notification(&BrokerNotification::Readiness(pending.notification()))?; + pending.confirm(); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -402,6 +420,28 @@ mod tests { drained } + #[test] + fn pending_readiness_survives_a_failed_nonblocking_send() { + struct Full(bool); + impl HostNotificationChannel for Full { + type Error = (); + fn send_notification(&mut self, _: &BrokerNotification) -> Result<(), ()> { + if self.0 { Err(()) } else { Ok(()) } + } + } + let publisher = ReadinessPublisher::new(); + publish(&publisher, HANDLE, ReadinessFlags::READ); + assert_eq!( + publish_pending_readiness(&publisher, &mut Full(true)), + Err(()) + ); + assert_eq!( + publish_pending_readiness(&publisher, &mut Full(false)), + Ok(()) + ); + assert!(publisher.take_pending().is_none()); + } + #[test] fn repeating_known_readiness_publishes_nothing() { let publisher = ReadinessPublisher::new(); diff --git a/litebox_broker_local_vm_userland/src/lib.rs b/litebox_broker_local_vm_userland/src/lib.rs index 8e6046a869..69641f99f8 100644 --- a/litebox_broker_local_vm_userland/src/lib.rs +++ b/litebox_broker_local_vm_userland/src/lib.rs @@ -10,6 +10,10 @@ //! (`BrokerEnter`'s Wake and Wait). The kernel serves the ring within each, //! so a call takes one entry. Shared memory is accessed only through //! `peer_memory`. +//! +//! Notifications (readiness of broker objects, e.g., standard input) have no +//! receiver thread: the runner receives them with [`Notifications::receive`], +//! which also waits for them. #![cfg(target_arch = "x86_64")] #![no_std] @@ -21,7 +25,8 @@ use alloc::sync::Arc; use litebox::utils::TruncateExt as _; use litebox_broker_local::BrokerLocal; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerRequest, BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, + BrokerResponse, }; use litebox_broker_protocol::wire::{self, WireError}; use litebox_broker_transport::channel::{LocalCallChannel, LocalSetupChannel}; @@ -98,8 +103,7 @@ impl LocalSetupChannel for KernelBrokerSetup { } } -/// The local ends of the request and response rings. The notification -/// consumer awaits the kernel's first notifying provider (e.g., timers). +/// The local ends of the request and response rings. struct Rings { requests: ControlRingProducer, responses: ControlRingConsumer, @@ -334,6 +338,13 @@ impl WaitableSharedMemory for KernelControlRing { } } +/// The notification ring's local end. +pub struct Notifications { + consumer: ControlRingConsumer, + /// For [`self_check`]. + words: KernelControlRing, +} + /// Call at most once per process. /// /// # Errors @@ -341,30 +352,81 @@ impl WaitableSharedMemory for KernelControlRing { /// The kernel rejects the association or negotiation fails. pub fn connect( info: &StartupInfo, -) -> litebox_broker_local::Result, ChannelError> { - let (local, _startup, ()) = BrokerLocal::negotiate(KernelBrokerSetup::default(), |_setup| { - let memory = KernelControlRing::new(info.broker_control_ring) - .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange))?; - let LocalControlRingEndpoints { - request_producer, - response_consumer, - // Unused until the kernel produces notifications. - notification_consumer: _, - } = ControlRing::new(memory) - .map_err(ChannelError::Ring)? - .into_local(); - let channel = KernelBrokerChannel { - rings: spin::Mutex::new(Some(Rings { - requests: request_producer, - responses: response_consumer, - })), - }; - let shared_memory: Arc = Arc::new(KernelSharedMemory { - region: info.broker_shared_memory, - }); - Ok((channel, shared_memory, ())) - })?; - Ok(local) +) -> litebox_broker_local::Result<(BrokerLocal, Notifications), ChannelError> { + let (local, _startup, notifications) = + BrokerLocal::negotiate(KernelBrokerSetup::default(), |_setup| { + let ring = || { + KernelControlRing::new(info.broker_control_ring) + .ok_or(ChannelError::Memory(SharedMemoryError::InvalidRange)) + }; + let LocalControlRingEndpoints { + request_producer, + response_consumer, + notification_consumer, + } = ControlRing::new(ring()?) + .map_err(ChannelError::Ring)? + .into_local(); + let notifications = Notifications { + consumer: notification_consumer, + words: ring()?, + }; + let channel = KernelBrokerChannel { + rings: spin::Mutex::new(Some(Rings { + requests: request_producer, + responses: response_consumer, + })), + }; + let shared_memory: Arc = Arc::new(KernelSharedMemory { + region: info.broker_shared_memory, + }); + Ok((channel, shared_memory, notifications)) + })?; + Ok((local, notifications)) +} + +impl Notifications { + /// Passes every published notification to `dispatch` (e.g., + /// `LiteBox::broker_notification_dispatcher`); if there is none, first + /// waits for one until `deadline` (TSC; zero for none). Returns how many + /// it passed: zero at the deadline. + /// + /// # Errors + /// + /// A malformed ring, or a wait the kernel rejects ([`Status::Stalled`] + /// if nothing could publish). + pub fn receive( + &mut self, + deadline: u64, + mut dispatch: impl FnMut(BrokerNotification), + ) -> Result { + let mut received = 0; + loop { + match self.consumer.try_read(wire::decode_notification) { + Ok(ControlRingReadStatus::Message(notification)) => { + dispatch(notification); + received += 1; + } + Ok(ControlRingReadStatus::Empty { wait_epoch }) => { + if received != 0 { + break; + } + let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + match broker_wait(epoch, wait_epoch, deadline) { + Ok(()) => {} + Err(Status::TimedOut) => return Ok(0), + Err(status) => return Err(ChannelError::Kernel(status)), + } + } + Err(ControlRingReadError::Ring(error)) => return Err(ChannelError::Ring(error)), + Err(ControlRingReadError::Decode(error)) => { + return Err(ChannelError::Wire(error)); + } + } + } + // The kernel sees the freed slots when it next publishes. + self.consumer.publish_head().map_err(ChannelError::Ring)?; + Ok(received) + } } /// Serves published broker requests. @@ -379,17 +441,40 @@ fn broker_wait(offset: usize, expected: u32, deadline: u64) -> Result<(), Status } /// Self-check (debug builds): the broker's futex operations, on the -/// notification ring's producer epoch, which stays zero (no notifications). +/// notification ring's producer epoch and on the request ring's consumer +/// epoch, which only the kernel's serving changes. /// /// # Panics /// /// On a failed check. -pub fn self_check() { +pub fn self_check(notifications: &Notifications, tsc_khz: u64) { let epoch = ControlRingDirection::Notifications.producer_epoch_offset(); + let served = ControlRingDirection::Requests.consumer_epoch_offset(); + // Safety: RDTSC has no side effects. + let now = || unsafe { core::arch::x86_64::_rdtsc() }; + let current = |offset| { + notifications + .words + .load_u32_acquire(offset) + .expect("a ring word") + }; broker_wake().expect("waking an idle broker"); - assert_eq!(broker_wait(epoch, 1, 0), Ok(()), "word differs"); - assert_eq!(broker_wait(epoch, 0, 0), Err(Status::Stalled)); - assert_eq!(broker_wait(epoch, 0, 1), Err(Status::Unsupported)); + let notified = current(epoch); + assert_eq!( + broker_wait(epoch, notified.wrapping_add(1), 0), + Ok(()), + "word differs" + ); + let soon = now() + tsc_khz; // 1 ms + assert_eq!( + broker_wait(served, current(served), 0), + Err(Status::Stalled) + ); + assert_eq!( + broker_wait(served, current(served), soon), + Err(Status::TimedOut) + ); + assert!(now() >= soon, "a timed wait returned early"); assert_eq!(broker_wait(0, 0, 0), Err(Status::InvalidArgument)); let mut wake = BrokerEnterRequest::wake(); wake.expected = 1; diff --git a/litebox_broker_vm_kernel/Cargo.toml b/litebox_broker_vm_kernel/Cargo.toml index 859e7c1bd0..1b9985c12b 100644 --- a/litebox_broker_vm_kernel/Cargo.toml +++ b/litebox_broker_vm_kernel/Cargo.toml @@ -14,6 +14,7 @@ litebox_common_vm_abi = { path = "../litebox_common_vm_abi", version = "0.1.0" } litebox_hal = { path = "../litebox_hal", version = "0.1.0" } litebox_platform_vm_kernel = { path = "../litebox_platform_vm_kernel", version = "0.1.0" } log = { version = "0.4", default-features = false } +spin = { version = "0.10.0", default-features = false, features = ["spin_mutex"] } [lints] workspace = true diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index a1ec7cd606..dcf38eedbe 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -7,6 +7,11 @@ //! responses are in the control ring, pinned and accessed through the //! kernel's mapping; payloads are in the process's lazily populated shared //! buffers. +//! +//! Files are the devices under `/dev`, with standard streams from the +//! runner's [`StdioProvider`]. Readiness (e.g., of standard input) goes to +//! the association's notification ring. Waiting ([`BrokerEnterOp::Wait`]) +//! halts until the runner's [`Events`] report something, e.g., an interrupt. #![cfg(target_arch = "x86_64")] #![no_std] @@ -16,6 +21,7 @@ extern crate alloc; mod memory; mod providers; +mod readiness; use alloc::boxed::Box; use alloc::sync::Arc; @@ -24,22 +30,23 @@ use core::cell::RefCell; use core::convert::Infallible; use litebox_broker_core::BrokerCore; use litebox_broker_core::readiness::ReadinessSink; +use litebox_broker_core::stdio::StdioProvider; use litebox_broker_host::BrokerHostAssociation; -use litebox_broker_protocol::ObjectHandle; use litebox_broker_protocol::message::{BrokerHandshakeRequest, BrokerHandshakeResponse}; -use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; use litebox_broker_protocol::wire; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::control_ring::{ BrokerControlRingEndpoints, CONTROL_RING_MEMORY_SIZE, ControlRing, ControlRingConsumer, - ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, MemoryAccessPolicy, + ControlRingDirection, ControlRingProducer, ControlRingReadStatus, ControlRingWriteStatus, + MemoryAccessPolicy, }; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool}; use litebox_common_vm_abi::{BrokerEnterOp, BrokerEnterRequest, Status, UserRange, WireFrame}; use litebox_platform_vm_kernel::PinnedUserPages; use memory::{PinnedControlRing, UserSharedMemory}; +use readiness::KernelReadiness; /// Size of a process's broker shared memory. pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; @@ -47,10 +54,31 @@ pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; /// Size of a process's broker control ring. pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; +/// What the broker waits for: the runner's event sources (e.g., device +/// interrupts and a timer). +pub trait Events: Sync { + /// Acts on events since the last call, e.g., publishes the readiness of + /// input that arrived. + fn poll(&self); + + /// Halts until an event may have happened, or until `deadline` (TSC) + /// passes; then [`Self::poll`] runs. Without a deadline, false if no + /// event could ever end the halt (and it does not halt). + fn wait(&self, deadline: Option) -> bool; +} + +/// The runner's sources for a [`Broker`]. +pub struct Config { + /// Behind `/dev/stdin`, `/dev/stdout`, and `/dev/stderr`. + pub stdio: Arc, + pub events: &'static dyn Events, +} + /// The kernel's broker; clones share it. #[derive(Clone)] pub struct Broker { core: BrokerCore, + events: &'static dyn Events, } impl Broker { @@ -59,13 +87,10 @@ impl Broker { /// # Panics /// /// Without a hardware CSPRNG, or on a second call. - #[expect( - clippy::new_without_default, - reason = "not `Default`: a second call panics" - )] - pub fn new() -> Self { + pub fn new(config: Config) -> Self { Self { - core: providers::core(), + core: providers::core(config.stdio), + events: config.events, } } @@ -87,40 +112,12 @@ impl Broker { Association { broker: self.clone(), shared_memory, + readiness: Arc::new(KernelReadiness::default()), state: RefCell::new(State::New(ring)), } } } -/// Discards readiness: the broker provides only randomness so far. A provider -/// with readiness (e.g., timers) needs a sink that publishes to the -/// notification ring. -struct NoReadiness; - -impl ReadinessSink for NoReadiness { - fn max_tracked_objects(&self) -> usize { - usize::MAX - } - - fn publish( - &self, - _handle: ObjectHandle, - _readiness: ReadinessFlags, - ) -> litebox_broker_core::Result<()> { - Ok(()) - } - - fn republish( - &self, - _handle: ObjectHandle, - _readiness: ReadinessFlags, - ) -> litebox_broker_core::Result<()> { - Ok(()) - } - - fn retire(&self, _handle: ObjectHandle) {} -} - /// One handshake request in, one response out. struct KernelHostSetup { request: Option, @@ -161,8 +158,8 @@ enum State { Ended, } -/// The broker's ends of the request and response rings. Its notification -/// producer awaits a provider with readiness (see `NoReadiness`). +/// The broker's ends of the request and response rings; the notification +/// ring's is in [`Association::readiness`]. struct Active { association: BrokerHostAssociation, requests: ControlRingConsumer, @@ -179,6 +176,7 @@ struct Active { pub struct Association { broker: Broker, shared_memory: UserRange, + readiness: Arc, state: RefCell, } @@ -211,7 +209,7 @@ impl Association { request: Some(request), response: None, }; - let readiness: Arc = Arc::new(NoReadiness); + let readiness: Arc = self.readiness.clone(); let result = litebox_broker_host::setup_connection( &self.broker.core, None, @@ -232,9 +230,9 @@ impl Association { let BrokerControlRingEndpoints { request_consumer, response_producer, - // Unused until a provider has readiness. - notification_producer: _, + notification_producer, } = ring.into_broker(); + self.readiness.activate(notification_producer); *state = State::Active(Box::new(Active { association: host, requests: request_consumer, @@ -252,6 +250,12 @@ impl Association { /// performs `request`'s operation. An invalid request has no effect. /// Before the handshake, any request is [`Status::Denied`]. /// + /// Waiting halts the kernel (single CPU, no scheduler) until the word + /// changes or the deadline passes. Only notifications change ring words + /// asynchronously; a wait on another word without a deadline, or on the + /// notification epoch when no event could publish, is + /// [`Status::Stalled`]. + /// /// # Errors /// /// See [`EnterError`]. @@ -269,9 +273,6 @@ impl Association { None } BrokerEnterOp::Wait => { - if request.deadline != 0 { - return Err(EnterError::Status(Status::Unsupported)); - } let offset = usize::try_from(request.offset).map_err(|_| invalid())?; if !MemoryAccessPolicy::ControlRing.permits_u32(offset) { return Err(invalid()); @@ -279,19 +280,37 @@ impl Association { Some(offset) } }; + self.broker.events.poll(); if active.serve().is_err() { if let State::Active(active) = core::mem::replace(&mut *state, State::Ended) { - end(active.association); + end(active.association, &self.readiness); } return Err(EnterError::Failed); } - // Nothing else changes ring words, so a word unchanged after serving - // stays so. - match wait_offset.map(|offset| active.words.load_u32_acquire(offset)) { - None => Ok(()), - Some(Ok(word)) if word != request.expected => Ok(()), - Some(Ok(_)) => Err(EnterError::Status(Status::Stalled)), - Some(Err(_)) => Err(invalid()), + self.readiness.flush(); + let Some(offset) = wait_offset else { + return Ok(()); + }; + let deadline = (request.deadline != 0).then_some(request.deadline); + let asynchronous = offset == ControlRingDirection::Notifications.producer_epoch_offset(); + loop { + match active.words.load_u32_acquire(offset) { + Ok(word) if word != request.expected => return Ok(()), + Ok(_) => {} + Err(_) => return Err(invalid()), + } + // Safety: RDTSC has no side effects. + if deadline.is_some_and(|deadline| unsafe { core::arch::x86_64::_rdtsc() } >= deadline) + { + return Err(EnterError::Status(Status::TimedOut)); + } + if deadline.is_none() && !asynchronous { + return Err(EnterError::Status(Status::Stalled)); + } + if !self.broker.events.wait(deadline) { + return Err(EnterError::Status(Status::Stalled)); + } + self.readiness.flush(); } } } @@ -369,7 +388,8 @@ pub enum EnterError { /// Ends an active association as the userland broker does when its runner /// goes away. -fn end(association: BrokerHostAssociation) { +fn end(association: BrokerHostAssociation, readiness: &KernelReadiness) { + readiness.close(); association.request_cancellation(); association.association_ending(); association.finish(); @@ -378,7 +398,7 @@ fn end(association: BrokerHostAssociation) { impl Drop for Association { fn drop(&mut self) { if let State::Active(active) = core::mem::replace(self.state.get_mut(), State::Ended) { - end(active.association); + end(active.association, &self.readiness); } } } diff --git a/litebox_broker_vm_kernel/src/providers.rs b/litebox_broker_vm_kernel/src/providers.rs index 9363dab670..6960c18d63 100644 --- a/litebox_broker_vm_kernel/src/providers.rs +++ b/litebox_broker_vm_kernel/src/providers.rs @@ -1,16 +1,19 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! The broker core's providers: hardware randomness only. +//! The broker core's providers: hardware randomness, and `/dev` (standard +//! streams from the runner's [`StdioProvider`], `null`, `urandom`). use alloc::sync::Arc; use litebox_broker_core::{ BrokerCore, ObjectRights, PolicyEngine, - fs::UnsupportedFileService, + fs::{FileService, composer::Composer, devices::Devices, resolver::Resolver}, random::{RandomProvider, RandomProviderError}, socket::UnsupportedSocketProvider, + stdio::StdioProvider, timer::UnsupportedTimerProvider, }; +use litebox_platform_vm_kernel::VmKernel; /// The HAL's hardware CSPRNG as the broker's randomness. struct HardwareRandom(litebox_hal::rng::Hardware); @@ -21,16 +24,31 @@ impl RandomProvider for HardwareRandom { } } +fn file_service( + stdio: Arc, + random: Arc, +) -> Arc { + let devices = Composer::builder() + .mount("/dev", |allocator| Devices::new(allocator, stdio, random)) + .build() + .unwrap_or_else(|error| panic!("broker file service: {error:?}")); + Arc::new(Resolver::::new(devices)) +} + +/// Runner processes, which the kernel vouches for, may open and use files; +/// the only ones are the devices. +/// /// # Panics /// /// Without a hardware CSPRNG, or on a second call. -pub(crate) fn core() -> BrokerCore { +pub(crate) fn core(stdio: Arc) -> BrokerCore { + let random: Arc = Arc::new(HardwareRandom(litebox_hal::rng::hardware())); BrokerCore::new( - PolicyEngine::with_host_guaranteed_rights(ObjectRights::empty()), + PolicyEngine::with_host_guaranteed_rights(ObjectRights::WAIT | ObjectRights::WRITE), Arc::new(UnsupportedSocketProvider), - Arc::new(HardwareRandom(litebox_hal::rng::hardware())), + Arc::clone(&random), Arc::new(UnsupportedTimerProvider), - Arc::new(UnsupportedFileService), + file_service(stdio, random), ) .unwrap_or_else(|error| panic!("broker core: {error:?}")) } diff --git a/litebox_broker_vm_kernel/src/readiness.rs b/litebox_broker_vm_kernel/src/readiness.rs new file mode 100644 index 0000000000..e31788ff38 --- /dev/null +++ b/litebox_broker_vm_kernel/src/readiness.rs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! An association's readiness, published to its (pinned) notification ring +//! at once, whichever process is current. What a full ring cannot take +//! stays pending until the next [`KernelReadiness::flush`]. + +use litebox_broker_core::readiness::ReadinessSink; +use litebox_broker_core::{BrokerError, Result}; +use litebox_broker_host::readiness::{ + MAX_TRACKED_READINESS_OBJECTS, ReadinessPublisher, publish_pending_readiness, +}; +use litebox_broker_protocol::ObjectHandle; +use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::readiness::ReadinessFlags; +use litebox_broker_protocol::wire; +use litebox_broker_transport::channel::HostNotificationChannel; +use litebox_broker_transport::control_ring::{ + ControlRingError, ControlRingProducer, ControlRingWriteStatus, +}; +use spin::mutex::SpinMutex; + +use crate::memory::PinnedControlRing; + +type Producer = ControlRingProducer; + +#[derive(Default)] +pub(crate) struct KernelReadiness { + publisher: ReadinessPublisher, + /// Until the handshake and after the association ends: `None`. + notifications: SpinMutex>, +} + +impl KernelReadiness { + /// Starts publishing to the ring, including what is already pending. + pub(crate) fn activate(&self, notifications: Producer) { + *self.notifications.lock() = Some(notifications); + self.flush(); + } + + /// Stops publishing, for good. + pub(crate) fn close(&self) { + self.publisher.close(); + *self.notifications.lock() = None; + } + + /// Publishes what is pending, as far as the ring has room. + pub(crate) fn flush(&self) { + // Nothing publishes with this lock held: sending only writes the + // ring, and interrupt handlers do not run sources. + let mut notifications = self.notifications.lock(); + let Some(producer) = notifications.as_mut() else { + return; + }; + match publish_pending_readiness(&self.publisher, &mut Ring(producer)) { + Ok(()) | Err(RingError::Full) => {} + Err(RingError::Ring(error)) => { + // The process corrupted its ring; its next broker entry fails + // the association. + log::warn!("broker notification ring failed: {error:?}"); + *notifications = None; + self.publisher.close(); + } + } + } +} + +enum RingError { + Full, + Ring(ControlRingError), +} + +/// Never blocks: a full ring fails the send. +struct Ring<'a>(&'a mut Producer); + +impl HostNotificationChannel for Ring<'_> { + type Error = RingError; + + fn send_notification( + &mut self, + notification: &BrokerNotification, + ) -> core::result::Result<(), RingError> { + let frame = wire::encode_notification(notification.clone()); + match self.0.try_write(&frame).map_err(RingError::Ring)? { + ControlRingWriteStatus::Written => Ok(()), + ControlRingWriteStatus::Full { .. } => Err(RingError::Full), + } + } +} + +impl ReadinessSink for KernelReadiness { + fn max_tracked_objects(&self) -> usize { + MAX_TRACKED_READINESS_OBJECTS + } + + fn publish(&self, handle: ObjectHandle, readiness: ReadinessFlags) -> Result<()> { + self.publisher + .publish(handle, readiness) + .map_err(|_| BrokerError::ResourceExhausted)?; + self.flush(); + Ok(()) + } + + fn republish(&self, handle: ObjectHandle, readiness: ReadinessFlags) -> Result<()> { + self.publisher + .republish(handle, readiness) + .map_err(|_| BrokerError::ResourceExhausted)?; + self.flush(); + Ok(()) + } + + fn retire(&self, handle: ObjectHandle) { + self.publisher.retire(handle); + } +} diff --git a/litebox_common_vm_abi/src/lib.rs b/litebox_common_vm_abi/src/lib.rs index f05077be5e..57b15c7c43 100644 --- a/litebox_common_vm_abi/src/lib.rs +++ b/litebox_common_vm_abi/src/lib.rs @@ -91,8 +91,9 @@ //! [`BrokerEnterOp::Wake`] and [`BrokerEnterOp::Wait`] on ring words. The //! kernel serves every published request first. //! - The ring's notification direction carries readiness of broker objects -//! (e.g., timers) from the kernel. The kernel's broker provides only -//! randomness so far, so it produces none yet. +//! (e.g., standard input) from the kernel, published as it changes; the +//! runner consumes them and waits for them on the direction's producer +//! epoch. //! - The kernel pins the control ring and accesses it through its own mapping. //! The shared buffers stay lazily populated and are touched only while //! executing a request that names them. @@ -591,8 +592,9 @@ pub enum BrokerEnterOp { Wake = 0, /// Returns once the `u32` ring word at `offset`, an epoch, differs from /// `expected`. [`Status::TimedOut`] at the deadline; [`Status::Stalled`] - /// if nothing could change the word. Deadlines are - /// [`Status::Unsupported`] so far. + /// if nothing could change the word. Only notifications change words + /// while the kernel waits (with no scheduler, it halts the CPU), so + /// without a deadline any other word stalls. Wait = 1, } diff --git a/litebox_hal/src/console.rs b/litebox_hal/src/console.rs index 8da1b619ff..f5081981ba 100644 --- a/litebox_hal/src/console.rs +++ b/litebox_hal/src/console.rs @@ -77,15 +77,19 @@ impl ComPort { } } - fn write_string(&mut self, s: &str) { + fn write_bytes(&mut self, bytes: &[u8]) { if !self.available { return; } - for byte in s.bytes() { + for &byte in bytes { self.write_byte(byte); } } + + fn write_string(&mut self, s: &str) { + self.write_bytes(s.as_bytes()); + } } static COM_ONCE: Once> = Once::new(); @@ -114,6 +118,11 @@ pub fn print_str(s: &str) { com().lock().write_string(s); } +/// Non-printable bytes show as `0xfe`. +pub fn print_bytes(bytes: &[u8]) { + com().lock().write_bytes(bytes); +} + /// Panic may interrupt a locked or initializing console. Never wait for it /// or alias its mutable handle; emergency output may interleave. pub fn panic_print(args: core::fmt::Arguments) { diff --git a/litebox_hal/src/dma.rs b/litebox_hal/src/dma.rs new file mode 100644 index 0000000000..d4bf2c72b0 --- /dev/null +++ b/litebox_hal/src/dma.rs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! What device drivers need from the kernel: memory a device can access +//! (DMA) and device registers (MMIO). The kernel platform implements +//! [`Hal`]. + +use core::ptr::NonNull; + +pub const PAGE_SIZE: usize = 4096; + +/// Zeroed, physically contiguous pages from a [`Hal`], which the kernel +/// accesses at `va` and devices at `pa`; returned to the [`Hal`] on drop, so +/// drop a region only once no device can access it (nothing in it is posted). +pub struct DmaRegion { + va: NonNull, + pa: u64, + len: usize, + hal: &'static dyn Hal, +} + +// Safety: a region is plain memory owned by whoever holds it. +unsafe impl Send for DmaRegion {} +// Safety: as above; shared access goes through raw pointers only. +unsafe impl Sync for DmaRegion {} + +impl DmaRegion { + /// At least `len` bytes; `None` when out of memory. + pub fn new(hal: &'static dyn Hal, len: usize) -> Option { + let pages = len.div_ceil(PAGE_SIZE).max(1); + let (va, pa) = hal.dma_alloc(pages)?; + let len = pages * PAGE_SIZE; + // Safety: freshly allocated and exclusively owned (`Hal` contract). + unsafe { va.write_bytes(0, len) }; + Some(Self { va, pa, len, hal }) + } + + pub fn pa(&self) -> u64 { + self.pa + } + + /// The kernel's pointer to byte `offset`. + /// + /// # Panics + /// + /// If `offset + size` exceeds the region. + pub fn ptr(&self, offset: usize, size: usize) -> *mut u8 { + assert!( + offset.checked_add(size).is_some_and(|end| end <= self.len), + "DMA access out of bounds" + ); + // Safety: within the region. + unsafe { self.va.as_ptr().add(offset) } + } +} + +impl Drop for DmaRegion { + fn drop(&mut self) { + // Safety: from `dma_alloc` with this many pages; see the type's docs + // for device access. + unsafe { self.hal.dma_dealloc(self.va, self.len / PAGE_SIZE) }; + } +} + +/// Device registers, mapped uncacheable. Accesses are volatile and +/// bounds-checked; offsets come from the driver, lengths from the device, +/// checked when the region is created. +#[derive(Clone, Copy, Debug)] +pub struct Mmio { + base: NonNull, + len: usize, +} + +// Safety: device registers have no Rust-visible state; drivers serialize +// their protocols themselves. +unsafe impl Send for Mmio {} +// Safety: as above. +unsafe impl Sync for Mmio {} + +macro_rules! mmio_accessors { + ($($read:ident $write:ident $ty:ty;)*) => {$( + pub fn $read(&self, offset: usize) -> $ty { + // Safety: in bounds and aligned (checked); a device register. + unsafe { self.at::<$ty>(offset).read_volatile() } + } + + pub fn $write(&self, offset: usize, value: $ty) { + // Safety: as above. + unsafe { self.at::<$ty>(offset).write_volatile(value) } + } + )*}; +} + +impl Mmio { + /// # Safety + /// + /// `base..base + len` must be device registers mapped uncacheable for the + /// result's lifetime and not used as memory by anything else. + pub unsafe fn new(base: NonNull, len: usize) -> Self { + Self { base, len } + } + + pub(crate) fn len(&self) -> usize { + self.len + } + + /// Device-supplied offsets are validated before they reach here, so an + /// out-of-bounds or misaligned access is a driver bug. + fn at(&self, offset: usize) -> *mut T { + assert!( + offset + .checked_add(size_of::()) + .is_some_and(|end| end <= self.len), + "MMIO access at {offset:#x} outside {:#x} bytes", + self.len + ); + // Safety: within the mapping. + let ptr: *mut T = unsafe { self.base.add(offset).cast().as_ptr() }; + // The address, not just the offset: a structure may start anywhere. + assert!(ptr.is_aligned(), "misaligned MMIO access at {ptr:p}"); + ptr + } + + mmio_accessors! { + read_u8 write_u8 u8; + read_u16 write_u16 u16; + read_u32 write_u32 u32; + } +} + +/// # Safety +/// +/// `dma_alloc` must return `pages` physically contiguous pages at `pa`, +/// mapped writable at the returned address and exclusively owned until +/// `dma_dealloc`. `map_mmio` must uphold [`Mmio::new`]'s contract. +pub unsafe trait Hal: Sync { + /// `None` when out of memory. + fn dma_alloc(&self, pages: usize) -> Option<(NonNull, u64)>; + + /// # Safety + /// + /// From `dma_alloc` with the same `pages`; no device accesses it. + unsafe fn dma_dealloc(&self, va: NonNull, pages: usize); + + /// Maps the device registers at physical `pa..pa + len` for the kernel's + /// lifetime; `None` if they cannot be (e.g., they overlap RAM). + /// + /// # Safety + /// + /// `pa..pa + len` must be device registers, e.g., within a BAR. + unsafe fn map_mmio(&self, pa: u64, len: usize) -> Option; +} + +#[cfg(test)] +mod tests { + use super::Mmio; + use core::ptr::NonNull; + + #[test] + #[should_panic(expected = "misaligned")] + fn misaligned_address_is_refused() { + let mut memory = [0u64; 4]; + // Safety: plain memory, alive for the test. + let mmio = unsafe { Mmio::new(NonNull::from(&mut memory).cast::().add(2), 30) }; + mmio.read_u32(0); + } +} diff --git a/litebox_hal/src/interrupt.rs b/litebox_hal/src/interrupt.rs index 23e234da77..b86aa8b63b 100644 --- a/litebox_hal/src/interrupt.rs +++ b/litebox_hal/src/interrupt.rs @@ -2,9 +2,36 @@ // Licensed under the MIT license. //! Interrupt controllers: the legacy 8259 PICs. +//! +//! Handlers mask their IRQ ([`mask_and_acknowledge`]) until the device is +//! serviced ([`unmask`]), as a level-triggered (PCI) line stays asserted +//! until then. Call with interrupts disabled: masks are read-modify-write. + +use x86_64::instructions::port::Port; const LEGACY_PIC_VECTOR_BASE: u8 = 0x20; +const MASTER_CMD: u16 = 0x20; +const MASTER_DATA: u16 = 0x21; +const SLAVE_CMD: u16 = 0xa0; +const SLAVE_DATA: u16 = 0xa1; +const CASCADE_IRQ: u8 = 2; +const EOI: u8 = 0x20; + +/// The vector of PIC IRQ `irq` (0..16). +pub const fn vector(irq: u8) -> u8 { + LEGACY_PIC_VECTOR_BASE + irq +} + +/// The PIC IRQ behind `vector`, if it is a PIC vector. +pub const fn irq(vector: u8) -> Option { + if vector >= LEGACY_PIC_VECTOR_BASE && vector < LEGACY_PIC_VECTOR_BASE + 16 { + Some(vector - LEGACY_PIC_VECTOR_BASE) + } else { + None + } +} + /// PIC IRQ7/IRQ15 can arrive while masked; SeaBIOS leaves LAPIC SVR at 0xff. pub const SPURIOUS_VECTORS: [u8; 3] = [ LEGACY_PIC_VECTOR_BASE + 7, @@ -14,11 +41,6 @@ pub const SPURIOUS_VECTORS: [u8; 3] = [ /// Must precede user entry (IF=1). PVH's PIC vectors overlap CPU exceptions. pub fn init_legacy_pics() { - use x86_64::instructions::port::Port; - const MASTER_CMD: u16 = 0x20; - const MASTER_DATA: u16 = 0x21; - const SLAVE_CMD: u16 = 0xa0; - const SLAVE_DATA: u16 = 0xa1; const ICW1_INIT_ICW4: u8 = 0x11; // edge-triggered, cascade, ICW4 follows const ICW4_8086: u8 = 0x01; // Safety: the standard 8259 initialization sequence; nothing else drives @@ -36,3 +58,43 @@ pub fn init_legacy_pics() { Port::::new(SLAVE_DATA).write(0xff); } } + +/// The mask register and bit of `irq`. +fn mask_register(irq: u8) -> (Port, u8) { + assert!(irq < 16, "PIC IRQs are 0..16"); + if irq < 8 { + (Port::new(MASTER_DATA), 1 << irq) + } else { + (Port::new(SLAVE_DATA), 1 << (irq - 8)) + } +} + +/// Lets `irq` interrupt (when IF=1); an IRQ on the slave also needs the +/// cascade. +pub fn unmask(irq: u8) { + let (mut port, bit) = mask_register(irq); + // Safety: only the PICs' mask registers change; nothing else drives the + // PICs, and the caller excludes interleaving (see the module docs). + unsafe { + let mask = port.read(); + port.write(mask & !bit); + } + if irq >= 8 { + unmask(CASCADE_IRQ); + } +} + +/// For interrupt handlers: masks `irq` until it is serviced, then ends the +/// interrupt at the PICs (non-specific EOI, to both PICs for a slave IRQ). +pub fn mask_and_acknowledge(irq: u8) { + let (mut port, bit) = mask_register(irq); + // Safety: as in `unmask`; an EOI only clears the PIC's in-service bit. + unsafe { + let mask = port.read(); + port.write(mask | bit); + if irq >= 8 { + Port::::new(SLAVE_CMD).write(EOI); + } + Port::::new(MASTER_CMD).write(EOI); + } +} diff --git a/litebox_hal/src/lib.rs b/litebox_hal/src/lib.rs index 12b88bc4db..9b3ac301df 100644 --- a/litebox_hal/src/lib.rs +++ b/litebox_hal/src/lib.rs @@ -9,14 +9,23 @@ #![no_std] #![warn(clippy::undocumented_unsafe_blocks)] +extern crate alloc; + #[cfg(target_arch = "x86_64")] pub mod clock; #[cfg(target_arch = "x86_64")] pub mod console; +pub mod dma; #[cfg(target_arch = "x86_64")] pub mod interrupt; #[cfg(target_arch = "x86_64")] +pub mod pci; +#[cfg(target_arch = "x86_64")] pub mod power; pub mod prk; #[cfg(target_arch = "x86_64")] pub mod rng; +#[cfg(target_arch = "x86_64")] +pub mod timer; +#[cfg(target_arch = "x86_64")] +pub mod virtio; diff --git a/litebox_hal/src/pci.rs b/litebox_hal/src/pci.rs new file mode 100644 index 0000000000..b5d262ae3d --- /dev/null +++ b/litebox_hal/src/pci.rs @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! PCI: configuration space through the legacy mechanism #1 ports +//! (`0xCF8`/`0xCFC`), on bus 0. Firmware (SeaBIOS under QEMU) has already +//! assigned BARs and routed INTx to a legacy PIC IRQ. + +use x86_64::instructions::port::Port; + +const CONFIG_ADDRESS: u16 = 0xcf8; +const CONFIG_DATA: u16 = 0xcfc; + +const VENDOR_ID: u8 = 0x00; +const DEVICE_ID: u8 = 0x02; +const SUBSYSTEM_ID: u8 = 0x2e; +const COMMAND: u8 = 0x04; +const STATUS: u8 = 0x06; +const HEADER_TYPE: u8 = 0x0e; +const CAPABILITIES_POINTER: u8 = 0x34; +const STATUS_CAPABILITIES: u16 = 1 << 4; +/// Capabilities live past the standard header. +const FIRST_CAPABILITY: u8 = 0x40; +const BAR0: u8 = 0x10; +const INTERRUPT_LINE: u8 = 0x3c; +const INTERRUPT_PIN: u8 = 0x3d; + +/// `COMMAND` bits. +pub const COMMAND_IO_SPACE: u16 = 1 << 0; +pub const COMMAND_MEMORY_SPACE: u16 = 1 << 1; +pub const COMMAND_BUS_MASTER: u16 = 1 << 2; +pub const COMMAND_INTX_DISABLE: u16 = 1 << 10; + +/// A function on bus 0. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Function { + pub device: u8, + pub function: u8, +} + +impl core::fmt::Display for Function { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "00:{:02x}.{}", self.device, self.function) + } +} + +impl Function { + fn select(self, offset: u8) { + let address = 0x8000_0000 + | (u32::from(self.device & 0x1f) << 11) + | (u32::from(self.function & 0x7) << 8) + | u32::from(offset & 0xfc); + // Safety: selects a configuration register; no memory side effects. + // Nothing else in the kernel accesses PCI configuration space, and + // callers run with interrupts disabled, so the address/data pair is + // not interleaved. + unsafe { Port::::new(CONFIG_ADDRESS).write(address) }; + } + + pub fn read_u32(self, offset: u8) -> u32 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA).read() } + } + + pub fn read_u16(self, offset: u8) -> u16 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 2)).read() } + } + + pub fn read_u8(self, offset: u8) -> u8 { + self.select(offset); + // Safety: see `select`. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 3)).read() } + } + + /// # Safety + /// + /// The write must not move a BAR or enable DMA into memory the caller + /// does not own. + pub unsafe fn write_u32(self, offset: u8, value: u32) { + self.select(offset); + // Safety: forwarded to the caller. + unsafe { Port::::new(CONFIG_DATA).write(value) }; + } + + pub fn vendor_id(self) -> u16 { + self.read_u16(VENDOR_ID) + } + + pub fn device_id(self) -> u16 { + self.read_u16(DEVICE_ID) + } + + pub fn subsystem_id(self) -> u16 { + self.read_u16(SUBSYSTEM_ID) + } + + pub fn command(self) -> u16 { + self.read_u16(COMMAND) + } + + /// A 16-bit write, leaving `STATUS` (write-1-to-clear) alone. + /// + /// # Safety + /// + /// Enabling bus mastering lets the device DMA wherever the driver points + /// it. + pub unsafe fn set_command(self, command: u16) { + self.select(COMMAND); + // Safety: forwarded to the caller. + unsafe { Port::::new(CONFIG_DATA + u16::from(COMMAND & 2)).write(command) }; + } + + /// Memory BAR `index` (with its upper half, if 64-bit), sized; `None` if + /// it is not a memory BAR or is unassigned. Sizing briefly disables the + /// function's decoding. + /// + /// # Safety + /// + /// Nothing may access the function's BARs meanwhile. + pub unsafe fn memory_bar(self, index: u8) -> Option { + if index >= 6 { + return None; + } + let offset = BAR0 + 4 * index; + let low = self.read_u32(offset); + let wide = (low >> 1) & 3 == 2; + if low & 1 != 0 || (wide && index == 5) { + return None; + } + let high = if wide { self.read_u32(offset + 4) } else { 0 }; + let command = self.command(); + // Safety: decoding is off while the BAR reads back its size mask, and + // both halves are restored before it is turned back on. + let (low_mask, high_mask) = unsafe { + self.set_command(command & !(COMMAND_IO_SPACE | COMMAND_MEMORY_SPACE)); + self.write_u32(offset, u32::MAX); + let low_mask = self.read_u32(offset); + self.write_u32(offset, low); + let high_mask = if wide { + self.write_u32(offset + 4, u32::MAX); + let mask = self.read_u32(offset + 4); + self.write_u32(offset + 4, high); + mask + } else { + u32::MAX + }; + self.set_command(command); + (low_mask, high_mask) + }; + let mask = (u64::from(high_mask) << 32) | u64::from(low_mask & !0xf); + let size = (!mask).wrapping_add(1); + let address = (u64::from(high) << 32) | u64::from(low & !0xf); + (size != 0 && address != 0 && address.checked_add(size).is_some()) + .then_some(MemoryBar { address, size }) + } + + /// Calls `f` with the offset and ID of each capability. The list comes + /// from the device: a malformed or cyclic one ends the walk. + pub fn for_each_capability(self, mut f: impl FnMut(u8, u8)) { + if self.read_u16(STATUS) & STATUS_CAPABILITIES == 0 { + return; + } + let mut offset = self.read_u8(CAPABILITIES_POINTER) & 0xfc; + // At most this many fit in configuration space. + for _ in 0..(256 - usize::from(FIRST_CAPABILITY)) / 4 { + if offset < FIRST_CAPABILITY { + return; + } + f(offset, self.read_u8(offset)); + offset = self.read_u8(offset + 1) & 0xfc; + } + } + + /// The legacy PIC IRQ that firmware routed INTx to, if any. + pub fn legacy_irq(self) -> Option { + if self.read_u8(INTERRUPT_PIN) == 0 { + return None; + } + Some(self.read_u8(INTERRUPT_LINE)).filter(|&irq| irq < 16) + } +} + +/// A memory BAR's physical range. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MemoryBar { + pub address: u64, + pub size: u64, +} + +/// The first function on bus 0 that `matches`. +pub fn find(matches: impl Fn(Function) -> bool) -> Option { + for device in 0..32 { + let first = Function { + device, + function: 0, + }; + if first.vendor_id() == 0xffff { + continue; + } + let multifunction = first.read_u8(HEADER_TYPE) & 0x80 != 0; + for function in 0..if multifunction { 8 } else { 1 } { + let candidate = Function { device, function }; + if matches(candidate) { + return Some(candidate); + } + } + } + None +} diff --git a/litebox_hal/src/timer.rs b/litebox_hal/src/timer.rs new file mode 100644 index 0000000000..f1547492f1 --- /dev/null +++ b/litebox_hal/src/timer.rs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Timer: PIT channel 0 in one-shot mode (mode 0), on PIC IRQ 0. Enough to +//! bound a halt; the TSC tells the time (see [`crate::clock`]). + +use x86_64::instructions::port::Port; + +/// The PIC IRQ the timer raises. +pub const IRQ: u8 = 0; + +const PIT_HZ: u64 = 1_193_182; +const PIT_CMD: u16 = 0x43; +const PIT_CH0: u16 = 0x40; +/// Channel 0, lobyte/hibyte, mode 0 (interrupt on terminal count), binary. +const CH0_ONESHOT: u8 = 0x30; + +/// Stops channel 0 without an interrupt; firmware may have left it +/// periodic. Call before unmasking [`IRQ`]. +pub fn disarm() { + // Safety: in mode 0, a control word holds OUT low and stops counting + // until a count is written. Nothing else uses channel 0. + unsafe { Port::::new(PIT_CMD).write(CH0_ONESHOT) }; +} + +/// Raises [`IRQ`] once, after `nanos`, clamped to about 55 ms (longer waits +/// re-arm). Replaces a pending shot. +pub fn arm_oneshot(nanos: u64) { + let ticks = u128::from(nanos) * u128::from(PIT_HZ) / 1_000_000_000; + let ticks = u16::try_from(ticks.clamp(1, 0xffff)).unwrap_or(u16::MAX); + let [lo, hi] = ticks.to_le_bytes(); + // Safety: programs channel 0 only; see `disarm`. + unsafe { + Port::::new(PIT_CMD).write(CH0_ONESHOT); + let mut ch0 = Port::::new(PIT_CH0); + ch0.write(lo); + ch0.write(hi); + } +} diff --git a/litebox_hal/src/virtio/console.rs b/litebox_hal/src/virtio/console.rs new file mode 100644 index 0000000000..183c27bec1 --- /dev/null +++ b/litebox_hal/src/virtio/console.rs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Virtio console (device type 3): QEMU's `virtio-serial-pci` with a +//! `virtconsole` on port 0, e.g.: +//! +//! ```text +//! -device virtio-serial-pci,id=vs0,disable-legacy=on +//! -chardev file,id=con0,path=out.txt,input-path=in.txt +//! -device virtconsole,chardev=con0,bus=vs0.0 +//! ``` +//! +//! Without `VIRTIO_CONSOLE_F_MULTIPORT` (not negotiated), the device is a +//! single port (port 0, where a `virtconsole` attaches) on queues 0 +//! (receive) and 1 (transmit), open without control-queue traffic. +//! +//! Neither direction blocks: [`VirtioConsole::write`] takes what free +//! transmit buffers hold, and [`VirtioConsole::read`] what has arrived. The +//! device interrupts when it returns buffers in either direction. + +use super::Error; +use super::pci::PciTransport; +use super::queue::{Buffer, VirtQueue}; +use crate::dma::{DmaRegion, Hal}; +use alloc::collections::VecDeque; +use alloc::vec; +use alloc::vec::Vec; + +const DEVICE_TYPE: u16 = 3; +const RECEIVE_QUEUE: u16 = 0; +const TRANSMIT_QUEUE: u16 = 1; + +const RECEIVE_BUFFERS: u16 = 16; +const RECEIVE_BUFFER_SIZE: usize = 256; +const TRANSMIT_BUFFERS: u16 = 32; +const TRANSMIT_BUFFER_SIZE: usize = 512; + +/// Fixed-size buffers in one DMA region, each a one-descriptor chain while +/// the device has it. +struct Buffers { + memory: DmaRegion, + size: usize, + /// By queue head: the buffer the device has. + by_head: Vec>, + free: Vec, +} + +impl Buffers { + fn new( + hal: &'static dyn Hal, + queue: &VirtQueue, + count: u16, + size: usize, + ) -> Result { + let count = count.min(queue.size()); + Ok(Self { + memory: DmaRegion::new(hal, usize::from(count) * size).ok_or(Error::OutOfMemory)?, + size, + by_head: vec![None; usize::from(queue.size())], + free: (0..count).rev().collect(), + }) + } + + fn ptr(&self, buffer: u16, offset: usize, len: usize) -> *mut u8 { + assert!(offset + len <= self.size); + self.memory + .ptr(usize::from(buffer) * self.size + offset, len) + } + + /// Gives a free buffer (`len` bytes of it) to the device. + fn post(&mut self, queue: &mut VirtQueue, buffer: u16, len: usize, device_writes: bool) { + let chain = [Buffer { + pa: self.memory.pa() + (usize::from(buffer) * self.size) as u64, + len: u32::try_from(len.min(self.size)).expect("buffers are small"), + device_writes, + }]; + // Safety: the buffer is ours (free) and DMA memory that lives as + // long as the console; it is not accessed until the device returns + // it. + let head = unsafe { queue.add(&chain) }.expect("a descriptor per buffer"); + self.by_head[usize::from(head)] = Some(buffer); + } + + /// The next buffer the device returned, with its length clamped. + fn take_used(&mut self, queue: &mut VirtQueue) -> Option<(u16, usize)> { + let (head, len) = queue.pop_used()?; + let buffer = self.by_head[usize::from(head)] + .take() + .expect("in-flight chains are posted buffers"); + Some(( + buffer, + usize::try_from(len).map_or(self.size, |len| len.min(self.size)), + )) + } +} + +/// A live console. Not internally synchronized. +pub struct VirtioConsole { + transport: PciTransport, + receive: VirtQueue, + receive_buffers: Buffers, + transmit: VirtQueue, + transmit_buffers: Buffers, + /// Filled receive buffers not yet read, as (buffer, length), oldest + /// first; `received_offset` bytes of the first are read. + received: VecDeque<(u16, usize)>, + received_offset: usize, +} + +impl VirtioConsole { + /// The first virtio console on bus 0, brought up; `None` if there is + /// none. + /// + /// # Errors + /// + /// The device cannot be driven. + pub fn probe(hal: &'static dyn Hal) -> Option> { + let function = super::find_pci_device(DEVICE_TYPE)?; + Some(PciTransport::new(hal, function).and_then(|transport| Self::bring_up(hal, transport))) + } + + fn bring_up(hal: &'static dyn Hal, transport: PciTransport) -> Result { + transport.init(0)?; + let setup = || -> Result<_, Error> { + let receive = transport.setup_queue(hal, RECEIVE_QUEUE, RECEIVE_BUFFERS)?; + let transmit = transport.setup_queue(hal, TRANSMIT_QUEUE, TRANSMIT_BUFFERS)?; + let receive_buffers = + Buffers::new(hal, &receive, RECEIVE_BUFFERS, RECEIVE_BUFFER_SIZE)?; + let transmit_buffers = + Buffers::new(hal, &transmit, TRANSMIT_BUFFERS, TRANSMIT_BUFFER_SIZE)?; + Ok((receive, receive_buffers, transmit, transmit_buffers)) + }; + let (mut receive, mut receive_buffers, transmit, transmit_buffers) = + setup().inspect_err(|_| transport.fail())?; + while let Some(buffer) = receive_buffers.free.pop() { + receive_buffers.post(&mut receive, buffer, RECEIVE_BUFFER_SIZE, true); + } + transport.start(); + transport.notify(&receive); + Ok(Self { + transport, + receive, + receive_buffers, + transmit, + transmit_buffers, + received: VecDeque::with_capacity(usize::from(RECEIVE_BUFFERS)), + received_offset: 0, + }) + } + + pub fn transport(&self) -> &PciTransport { + &self.transport + } + + /// Queues as much of `bytes` as free transmit buffers hold; returns how + /// much. Zero means every buffer is still with the device. + pub fn write(&mut self, bytes: &[u8]) -> usize { + while let Some((buffer, _)) = self.transmit_buffers.take_used(&mut self.transmit) { + self.transmit_buffers.free.push(buffer); + } + let mut written = 0; + while written < bytes.len() { + let Some(buffer) = self.transmit_buffers.free.pop() else { + break; + }; + let chunk = &bytes[written..]; + let chunk = &chunk[..chunk.len().min(TRANSMIT_BUFFER_SIZE)]; + let destination = self.transmit_buffers.ptr(buffer, 0, chunk.len()); + // Safety: a free buffer, which the device does not access. + unsafe { core::ptr::copy_nonoverlapping(chunk.as_ptr(), destination, chunk.len()) }; + self.transmit_buffers + .post(&mut self.transmit, buffer, chunk.len(), false); + written += chunk.len(); + } + if written != 0 { + self.transport.notify(&self.transmit); + } + written + } + + /// Whether [`Self::read`] would return data. + pub fn has_input(&mut self) -> bool { + self.collect_received(); + !self.received.is_empty() + } + + /// Copies received bytes into `output`; returns how many. + pub fn read(&mut self, output: &mut [u8]) -> usize { + self.collect_received(); + let mut read = 0; + let mut reposted = false; + while read < output.len() { + let Some(&(buffer, len)) = self.received.front() else { + break; + }; + let n = (len - self.received_offset).min(output.len() - read); + let source = self.receive_buffers.ptr(buffer, self.received_offset, n); + // Safety: the device returned the buffer; `n` bytes are in it. + unsafe { core::ptr::copy_nonoverlapping(source, output[read..].as_mut_ptr(), n) }; + read += n; + self.received_offset += n; + if self.received_offset == len { + self.received.pop_front(); + self.received_offset = 0; + self.receive_buffers + .post(&mut self.receive, buffer, RECEIVE_BUFFER_SIZE, true); + reposted = true; + } + } + if reposted { + self.transport.notify(&self.receive); + } + read + } + + fn collect_received(&mut self) { + let mut reposted = false; + while let Some((buffer, len)) = self.receive_buffers.take_used(&mut self.receive) { + if len == 0 { + self.receive_buffers + .post(&mut self.receive, buffer, RECEIVE_BUFFER_SIZE, true); + reposted = true; + } else { + self.received.push_back((buffer, len)); + } + } + if reposted { + self.transport.notify(&self.receive); + } + } +} diff --git a/litebox_hal/src/virtio/mod.rs b/litebox_hal/src/virtio/mod.rs new file mode 100644 index 0000000000..9875a84367 --- /dev/null +++ b/litebox_hal/src/virtio/mod.rs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Virtio 1.x devices over PCI ([`pci::PciTransport`]): the modern +//! interface, with its structures in memory BARs (mapped through +//! [`crate::dma::Hal`]) and INTx interrupts (routed to a legacy PIC IRQ by +//! firmware; MSI-X is left disabled). Split virtqueues ([`queue`]). +//! +//! Devices: [`console`]. The transport and queues are device-independent, +//! for further devices (e.g., file systems and networking for the broker). +//! +//! Devices are not trusted: the capability list, BAR ranges, queue sizes and +//! used-ring entries are checked, and lengths the device reports are clamped +//! by the driver before use. + +pub mod console; +pub mod pci; +pub mod queue; + +const PCI_VENDOR: u16 = 0x1af4; + +/// The first virtio function of type `device_type` on bus 0. Modern-only +/// device IDs are 0x1040 + type; transitional ones (0x1000..=0x103f) give +/// the type in the subsystem ID. +pub fn find_pci_device(device_type: u16) -> Option { + crate::pci::find(|function| { + let device_id = function.device_id(); + function.vendor_id() == PCI_VENDOR + && (device_id == 0x1040 + device_type + || ((0x1000..=0x103f).contains(&device_id) + && function.subsystem_id() == device_type)) + }) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// The device lacks a required structure (common, notify, or ISR). + MissingStructure(&'static str), + /// A structure is outside its BAR, too short, or not in a memory BAR. + BadStructure(&'static str), + /// The platform could not map a structure. + MapFailed, + ResetTimedOut, + /// No `VIRTIO_F_VERSION_1`. + NotModern, + /// The device cleared `FEATURES_OK`. + FeaturesRejected, + QueueUnavailable(u16), + /// The queue's notification address is outside the notify structure. + BadNotifyOffset(u16), + OutOfMemory, +} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::MissingStructure(what) => write!(f, "no {what} structure"), + Self::BadStructure(what) => write!(f, "malformed {what} structure"), + Self::MapFailed => f.write_str("cannot map device registers"), + Self::ResetTimedOut => f.write_str("reset timed out"), + Self::NotModern => f.write_str("VIRTIO_F_VERSION_1 not offered"), + Self::FeaturesRejected => f.write_str("features rejected"), + Self::QueueUnavailable(queue) => write!(f, "queue {queue} unavailable"), + Self::BadNotifyOffset(queue) => write!(f, "queue {queue} notify offset out of range"), + Self::OutOfMemory => f.write_str("out of DMA memory"), + } + } +} diff --git a/litebox_hal/src/virtio/pci.rs b/litebox_hal/src/virtio/pci.rs new file mode 100644 index 0000000000..bd286ef2e3 --- /dev/null +++ b/litebox_hal/src/virtio/pci.rs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! The modern virtio-PCI transport: structures located by vendor-specific +//! capabilities, in memory BARs. + +use super::Error; +use super::queue::VirtQueue; +use crate::dma::{Hal, Mmio}; +use crate::pci::{self, Function, MemoryBar}; + +const CAP_VENDOR_SPECIFIC: u8 = 0x09; + +/// `VIRTIO_F_VERSION_1`: the modern interface; always negotiated. +const F_VERSION_1: u64 = 1 << 32; + +/// `virtio_pci_cap` fields, from the capability's offset. +mod cap { + pub const CFG_TYPE: u8 = 3; + pub const BAR: u8 = 4; + pub const OFFSET: u8 = 8; + pub const LENGTH: u8 = 12; + /// `virtio_pci_notify_cap` only. + pub const NOTIFY_OFF_MULTIPLIER: u8 = 16; + /// Of a `virtio_pci_notify_cap`, the longest one read. + pub const LEN: u8 = 20; + + pub const COMMON: u8 = 1; + pub const NOTIFY: u8 = 2; + pub const ISR: u8 = 3; + pub const DEVICE: u8 = 4; +} + +/// `virtio_pci_common_cfg` fields. +mod common { + pub const DEVICE_FEATURE_SELECT: usize = 0x00; + pub const DEVICE_FEATURE: usize = 0x04; + pub const DRIVER_FEATURE_SELECT: usize = 0x08; + pub const DRIVER_FEATURE: usize = 0x0c; + pub const DEVICE_STATUS: usize = 0x14; + pub const QUEUE_SELECT: usize = 0x16; + pub const QUEUE_SIZE: usize = 0x18; + pub const QUEUE_ENABLE: usize = 0x1c; + pub const QUEUE_NOTIFY_OFF: usize = 0x1e; + pub const QUEUE_DESC: usize = 0x20; + pub const QUEUE_DRIVER: usize = 0x28; + pub const QUEUE_DEVICE: usize = 0x30; + pub const LEN: usize = 0x38; +} + +mod status { + pub const ACKNOWLEDGE: u8 = 1; + pub const DRIVER: u8 = 2; + pub const DRIVER_OK: u8 = 4; + pub const FEATURES_OK: u8 = 8; + pub const FAILED: u8 = 0x80; +} + +const RESET_POLL_LIMIT: u32 = 1_000_000; + +/// A structure's location, from its capability. +#[derive(Clone, Copy)] +struct Location { + bar: u8, + offset: u32, + length: u32, +} + +/// The device's capabilities: the first of each structure type, as the +/// specification asks drivers to prefer. +#[derive(Default)] +struct Locations { + common: Option, + notify: Option<(Location, u32)>, + isr: Option, + device: Option, +} + +impl Locations { + fn read(function: Function) -> Self { + let mut found = Self::default(); + function.for_each_capability(|offset, id| { + // Every field read must be inside configuration space. + if id != CAP_VENDOR_SPECIFIC || offset.checked_add(cap::LEN).is_none() { + return; + } + let location = Location { + bar: function.read_u8(offset + cap::BAR), + offset: function.read_u32(offset + cap::OFFSET), + length: function.read_u32(offset + cap::LENGTH), + }; + match function.read_u8(offset + cap::CFG_TYPE) { + cap::COMMON => { + found.common.get_or_insert(location); + } + cap::NOTIFY => { + let multiplier = function.read_u32(offset + cap::NOTIFY_OFF_MULTIPLIER); + found.notify.get_or_insert((location, multiplier)); + } + cap::ISR => { + found.isr.get_or_insert(location); + } + cap::DEVICE => { + found.device.get_or_insert(location); + } + _ => {} + } + }); + found + } +} + +/// A virtio PCI function's mapped structures. +pub struct PciTransport { + function: Function, + common: Mmio, + notify: Mmio, + notify_off_multiplier: u32, + isr: Mmio, + device: Option, + irq: Option, +} + +impl PciTransport { + /// Maps `function`'s structures and enables its memory decoding, bus + /// mastering, and INTx. + /// + /// # Errors + /// + /// A missing or malformed structure, or one that cannot be mapped. + pub fn new(hal: &dyn Hal, function: Function) -> Result { + let locations = Locations::read(function); + let mut bars = [None::; 6]; + let mut map = |location: Location, what, min_len: usize| -> Result { + let bad = Error::BadStructure(what); + let slot = bars.get_mut(usize::from(location.bar)).ok_or(bad)?; + let bar = match *slot { + Some(bar) => bar, + // Safety: the driver does not access the BARs yet. + None => *slot.insert(unsafe { function.memory_bar(location.bar) }.ok_or(bad)?), + }; + let (offset, length) = (u64::from(location.offset), u64::from(location.length)); + if length < min_len as u64 || offset + length > bar.size { + return Err(bad); + } + let length = usize::try_from(length).map_err(|_| bad)?; + // Safety: within the BAR, so device registers. + unsafe { hal.map_mmio(bar.address + offset, length) }.ok_or(Error::MapFailed) + }; + let common = locations.common.ok_or(Error::MissingStructure("common"))?; + let (notify, notify_off_multiplier) = + locations.notify.ok_or(Error::MissingStructure("notify"))?; + let isr = locations.isr.ok_or(Error::MissingStructure("ISR"))?; + let transport = Self { + function, + common: map(common, "common", common::LEN)?, + notify: map(notify, "notify", 2)?, + notify_off_multiplier, + isr: map(isr, "ISR", 1)?, + device: locations + .device + .map(|device| map(device, "device", 0)) + .transpose()?, + irq: function.legacy_irq(), + }; + let command = (function.command() | pci::COMMAND_MEMORY_SPACE | pci::COMMAND_BUS_MASTER) + & !pci::COMMAND_INTX_DISABLE; + // Safety: decodes the BARs firmware assigned; DMA only goes where the + // driver points the queues. + unsafe { function.set_command(command) }; + Ok(transport) + } + + pub fn function(&self) -> Function { + self.function + } + + /// The PIC IRQ of the device's INTx, if firmware routed one. + pub fn irq(&self) -> Option { + self.irq + } + + /// The device-specific configuration, if any. + pub fn device_config(&self) -> Option { + self.device + } + + fn status(&self) -> u8 { + self.common.read_u8(common::DEVICE_STATUS) + } + + fn set_status(&self, status: u8) { + self.common.write_u8(common::DEVICE_STATUS, status); + } + + fn add_status(&self, bits: u8) { + self.set_status(self.status() | bits); + } + + /// Marks the device `FAILED`, e.g., after a failed initialization. + pub fn fail(&self) { + self.add_status(status::FAILED); + } + + /// Resets the device and negotiates `wanted` features (plus + /// `VIRTIO_F_VERSION_1`) as far as it offers them; returns those. Set up + /// queues next, then [`Self::start`]. + /// + /// # Errors + /// + /// The device did not reset, is not modern, or rejected the features. It + /// is left `FAILED`, except after a reset timeout. + pub fn init(&self, wanted: u64) -> Result { + self.set_status(0); + if !(0..RESET_POLL_LIMIT).any(|_| self.status() == 0) { + return Err(Error::ResetTimedOut); + } + self.add_status(status::ACKNOWLEDGE); + self.add_status(status::DRIVER); + let offered = (0..2u32).fold(0, |features, word| { + self.common.write_u32(common::DEVICE_FEATURE_SELECT, word); + features | u64::from(self.common.read_u32(common::DEVICE_FEATURE)) << (32 * word) + }); + if offered & F_VERSION_1 == 0 { + self.fail(); + return Err(Error::NotModern); + } + let accepted = offered & (wanted | F_VERSION_1); + for word in 0..2u32 { + self.common.write_u32(common::DRIVER_FEATURE_SELECT, word); + #[expect(clippy::cast_possible_truncation, reason = "one 32-bit word")] + self.common + .write_u32(common::DRIVER_FEATURE, (accepted >> (32 * word)) as u32); + } + self.add_status(status::FEATURES_OK); + if self.status() & status::FEATURES_OK == 0 { + self.fail(); + return Err(Error::FeaturesRejected); + } + Ok(accepted) + } + + /// Sets up and enables queue `index`, with at most `max_size` entries. + /// + /// # Errors + /// + /// The device lacks the queue, its notification address is out of + /// range, or memory ran out. + pub fn setup_queue( + &self, + hal: &'static dyn Hal, + index: u16, + max_size: u16, + ) -> Result { + self.common.write_u16(common::QUEUE_SELECT, index); + let device_max = self.common.read_u16(common::QUEUE_SIZE); + // Split queues of a power-of-two size work with every device. + let size = device_max.min(max_size); + if size == 0 { + return Err(Error::QueueUnavailable(index)); + } + let size = 1 << size.ilog2(); + let notify_offset = usize::from(self.common.read_u16(common::QUEUE_NOTIFY_OFF)) + .checked_mul(usize::try_from(self.notify_off_multiplier).unwrap_or(usize::MAX)) + .filter(|&offset| offset.is_multiple_of(2) && offset + 2 <= self.notify.len()) + .ok_or(Error::BadNotifyOffset(index))?; + let queue = VirtQueue::new(hal, index, size, notify_offset)?; + let [desc, driver, device] = queue.addresses(); + self.common.write_u16(common::QUEUE_SIZE, size); + for (field, address) in [ + (common::QUEUE_DESC, desc), + (common::QUEUE_DRIVER, driver), + (common::QUEUE_DEVICE, device), + ] { + #[expect(clippy::cast_possible_truncation, reason = "split into words")] + { + self.common.write_u32(field, address as u32); + self.common.write_u32(field + 4, (address >> 32) as u32); + } + } + self.common.write_u16(common::QUEUE_ENABLE, 1); + Ok(queue) + } + + /// Lets the device use its queues. + pub fn start(&self) { + self.add_status(status::DRIVER_OK); + } + + /// Tells the device that `queue` has new available buffers. + pub fn notify(&self, queue: &VirtQueue) { + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + self.notify.write_u16(queue.notify_offset(), queue.index()); + } + + /// Acknowledges the device's interrupt (deasserting INTx); returns the + /// ISR status (bit 0: a queue, bit 1: configuration). Process the queues + /// afterwards, so nothing that arrived before goes unnoticed. + pub fn acknowledge_interrupt(&self) -> u8 { + self.isr.read_u8(0) + } +} diff --git a/litebox_hal/src/virtio/queue.rs b/litebox_hal/src/virtio/queue.rs new file mode 100644 index 0000000000..683ad6f442 --- /dev/null +++ b/litebox_hal/src/virtio/queue.rs @@ -0,0 +1,282 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Split virtqueues. A request is a chain of buffers (device-readable, then +//! device-writable), identified by its head descriptor until the device +//! returns it. The driver keeps its own record of chains and the free list, +//! so a device that writes nonsense into the used ring cannot corrupt them. + +use super::Error; +use crate::dma::{DmaRegion, Hal}; +use alloc::vec; +use alloc::vec::Vec; +use core::sync::atomic::{Ordering, fence}; + +const DESC_F_NEXT: u16 = 1; +const DESC_F_WRITE: u16 = 2; +const DESC_SIZE: usize = 16; + +/// One buffer of a chain, in DMA memory. +#[derive(Clone, Copy, Debug)] +pub struct Buffer { + pub pa: u64, + pub len: u32, + /// The device writes it (else reads it). Device-writable buffers follow + /// all device-readable ones in a chain. + pub device_writes: bool, +} + +pub struct VirtQueue { + index: u16, + size: u16, + notify_offset: usize, + /// Descriptors, then the available ring; the used ring at `used`. + ring: DmaRegion, + avail: usize, + used: usize, + /// Private copies of each descriptor's `next`, threading the free list + /// and in-flight chains. + next: Vec, + /// By head: the length of an in-flight chain, else zero. + in_flight: Vec, + free_head: u16, + num_free: u16, + next_avail: u16, + last_used: u16, +} + +// Safety: the queue exclusively owns its ring. +unsafe impl Send for VirtQueue {} + +impl VirtQueue { + /// `size` must be a power of two; see `PciTransport::setup_queue`. + pub(super) fn new( + hal: &'static dyn Hal, + index: u16, + size: u16, + notify_offset: usize, + ) -> Result { + assert!(size.is_power_of_two()); + let n = usize::from(size); + // Alignments: descriptors 16, available ring 2, used ring 4. + let avail = DESC_SIZE * n; + let used = (avail + 6 + 2 * n).next_multiple_of(4); + let ring = DmaRegion::new(hal, used + 6 + 8 * n).ok_or(Error::OutOfMemory)?; + Ok(Self { + index, + size, + notify_offset, + ring, + avail, + used, + next: (1..=size).collect(), + in_flight: vec![0; n], + free_head: 0, + num_free: size, + next_avail: 0, + last_used: 0, + }) + } + + pub fn index(&self) -> u16 { + self.index + } + + pub fn size(&self) -> u16 { + self.size + } + + pub(super) fn notify_offset(&self) -> usize { + self.notify_offset + } + + /// The descriptor table's, available ring's, and used ring's addresses. + pub(super) fn addresses(&self) -> [u64; 3] { + let pa = self.ring.pa(); + [pa, pa + self.avail as u64, pa + self.used as u64] + } + + fn write(&self, offset: usize, value: T) { + // Safety: in bounds (checked by `ptr`) and naturally aligned (the + // layout keeps every field so); the driver's part of the ring. + unsafe { + self.ring + .ptr(offset, size_of::()) + .cast::() + .write_volatile(value); + } + } + + fn read(&self, offset: usize) -> T { + // Safety: as for `write`; the device's part is read volatile. + unsafe { + self.ring + .ptr(offset, size_of::()) + .cast::() + .read_volatile() + } + } + + /// Makes `chain` available to the device; returns its head, or `None` if + /// it is empty or there are not enough free descriptors. The device + /// learns of it at the next [`super::pci::PciTransport::notify`]. + /// + /// # Safety + /// + /// The buffers must stay valid DMA memory, accessed only as the device + /// permits, until the device returns the chain ([`Self::pop_used`]). + pub unsafe fn add(&mut self, chain: &[Buffer]) -> Option { + let len = u16::try_from(chain.len()).ok()?; + if len == 0 || len > self.num_free { + return None; + } + let head = self.free_head; + let mut id = head; + for (i, buffer) in chain.iter().enumerate() { + let desc = usize::from(id) * DESC_SIZE; + let next = self.next[usize::from(id)]; + let mut flags = if buffer.device_writes { + DESC_F_WRITE + } else { + 0 + }; + if i + 1 < chain.len() { + flags |= DESC_F_NEXT; + } + self.write(desc, buffer.pa); + self.write(desc + 8, buffer.len); + self.write(desc + 12, flags); + self.write(desc + 14, next); + if i + 1 < chain.len() { + id = next; + } + } + self.free_head = self.next[usize::from(id)]; + self.num_free -= len; + self.in_flight[usize::from(head)] = len; + let slot = self.avail + 4 + 2 * usize::from(self.next_avail % self.size); + self.write(slot, head); + // The device must see the descriptors and slot before the index. + fence(Ordering::SeqCst); + self.next_avail = self.next_avail.wrapping_add(1); + self.write(self.avail + 2, self.next_avail); + Some(head) + } + + /// The next chain the device returned: its head, and how many bytes the + /// device says it wrote (untrusted; clamp before use). Entries for + /// chains that are not in flight are skipped. + pub fn pop_used(&mut self) -> Option<(u16, u32)> { + loop { + let used_idx: u16 = self.read(self.used + 2); + if used_idx == self.last_used { + return None; + } + fence(Ordering::Acquire); + let elem = self.used + 4 + 8 * usize::from(self.last_used % self.size); + let (id, len): (u32, u32) = (self.read(elem), self.read(elem + 4)); + self.last_used = self.last_used.wrapping_add(1); + let Some(head) = u16::try_from(id).ok().filter(|&id| id < self.size) else { + continue; + }; + let chain_len = core::mem::take(&mut self.in_flight[usize::from(head)]); + if chain_len == 0 { + continue; + } + let mut tail = head; + for _ in 1..chain_len { + tail = self.next[usize::from(tail)]; + } + self.next[usize::from(tail)] = self.free_head; + self.free_head = head; + self.num_free += chain_len; + return Some((head, len)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dma::Mmio; + + /// Heap memory, with physical = virtual addresses. + struct HeapHal; + + fn layout(pages: usize) -> core::alloc::Layout { + core::alloc::Layout::from_size_align(pages * crate::dma::PAGE_SIZE, 4096).unwrap() + } + + // Safety: heap memory is contiguous and exclusively owned. + unsafe impl Hal for HeapHal { + fn dma_alloc(&self, pages: usize) -> Option<(core::ptr::NonNull, u64)> { + // Safety: non-zero size. + let va = core::ptr::NonNull::new(unsafe { alloc::alloc::alloc(layout(pages)) })?; + Some((va, va.as_ptr() as u64)) + } + + unsafe fn dma_dealloc(&self, va: core::ptr::NonNull, pages: usize) { + // Safety: from `dma_alloc` with the same layout. + unsafe { alloc::alloc::dealloc(va.as_ptr(), layout(pages)) }; + } + + unsafe fn map_mmio(&self, _pa: u64, _len: usize) -> Option { + None + } + } + + fn queue(size: u16) -> VirtQueue { + VirtQueue::new(&HeapHal, 0, size, 0).unwrap() + } + + /// Plays the device: returns `id` with `len` in the used ring. + fn device_returns(queue: &VirtQueue, slot: u16, id: u32, len: u32) { + let elem = queue.used + 4 + 8 * usize::from(slot % queue.size); + queue.write(elem, id); + queue.write(elem + 4, len); + queue.write(queue.used + 2, slot.wrapping_add(1)); + } + + fn buffer(device_writes: bool) -> Buffer { + Buffer { + pa: 0x1000, + len: 8, + device_writes, + } + } + + #[test] + fn chains_are_published_and_freed() { + let mut q = queue(4); + // Safety: the device never touches the (fake) buffers. + let head = unsafe { q.add(&[buffer(false), buffer(true)]) }.unwrap(); + assert_eq!(q.num_free, 2); + let avail_idx: u16 = q.read(q.avail + 2); + assert_eq!(avail_idx, 1); + let flags: u16 = q.read(usize::from(head) * DESC_SIZE + 12); + assert_eq!(flags, DESC_F_NEXT); + assert_eq!(q.pop_used(), None); + device_returns(&q, 0, u32::from(head), 5); + assert_eq!(q.pop_used(), Some((head, 5))); + assert_eq!(q.num_free, 4); + } + + #[test] + fn bogus_and_replayed_used_entries_are_skipped() { + let mut q = queue(4); + // Safety: as above. + let head = unsafe { q.add(&[buffer(true)]) }.unwrap(); + device_returns(&q, 0, 1000, 1); // out of range + assert_eq!(q.pop_used(), None); + device_returns(&q, 1, u32::from(head) + 1, 1); // not in flight + assert_eq!(q.pop_used(), None); + device_returns(&q, 2, u32::from(head), 1); + assert_eq!(q.pop_used(), Some((head, 1))); + device_returns(&q, 3, u32::from(head), 1); // replayed + assert_eq!(q.pop_used(), None); + assert_eq!(q.num_free, 4); + // The free list is intact: the whole queue can be used again. + // Safety: as above. + assert!(unsafe { q.add(&[buffer(true); 4]) }.is_some()); + } +} diff --git a/litebox_platform_vm_kernel/Cargo.toml b/litebox_platform_vm_kernel/Cargo.toml index dc9e9fce6a..555d12ac62 100644 --- a/litebox_platform_vm_kernel/Cargo.toml +++ b/litebox_platform_vm_kernel/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] litebox = { path = "../litebox/", version = "0.1.0" } litebox_common_linux = { path = "../litebox_common_linux/", version = "0.1.0" } +litebox_hal = { path = "../litebox_hal", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } spin = { version = "0.10.0", default-features = false, features = [ diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.S b/litebox_platform_vm_kernel/src/arch/x86/interrupts.S index f68070dbda..2f861e4cc6 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.S +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.S @@ -143,26 +143,57 @@ isr_ignore: iretq /* - * IDT entries require an 8-byte stride; CALL pushes stub + 5 for vector lookup. - * Kernel-fatal even from user mode; GS may still belong to user code. + * External interrupts (vectors 32..=255), handled by the registered handler + * or fatal. IDT entries require an 8-byte stride; CALL pushes stub + 5 for + * vector lookup. + * + * They arrive from user mode, or from the kernel only while it halts with + * IF=1. Entry does not depend on GS (which may still be the user's): the + * handler must not touch per-CPU data, so no SWAPGS. The kernel runs with + * SSE disabled, so only the caller-saved GPRs need saving. */ .balign 8 -.global isr_unexpected_stubs -isr_unexpected_stubs: +.global isr_external_stubs +isr_external_stubs: .rept 224 .balign 8 - call isr_unexpected + call isr_external .endr -isr_unexpected: +isr_external: cld - pop rdi /* stub return address */ - lea rax, [rip + isr_unexpected_stubs + 5] + clac /* SMAP stays enforced; IRETQ restores AC */ + /* [rsp] stub return address, [rsp+8] RIP, [rsp+16] CS */ + push rax + push rcx + push rdx + push rsi + push rdi + push r8 + push r9 + push r10 + push r11 + push rbp + mov rbp, rsp + mov rdi, [rbp + 80] /* stub return address */ + lea rax, [rip + isr_external_stubs + 5] sub rdi, rax shr rdi, 3 add rdi, 32 /* vector */ - mov rsi, [rsp] /* interrupted RIP */ - mov rdx, [rsp + 8] /* interrupted CS */ + mov rsi, [rbp + 88] /* interrupted RIP */ + mov rdx, [rbp + 96] /* interrupted CS */ and rsp, -16 - call unexpected_interrupt_handler_impl - ud2 + call external_interrupt_handler_impl + mov rsp, rbp + pop rbp + pop r11 + pop r10 + pop r9 + pop r8 + pop rdi + pop rsi + pop rdx + pop rcx + pop rax + add rsp, 8 /* stub return address */ + iretq diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs index e3835e3a69..5536200824 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs @@ -5,6 +5,7 @@ //! NMI entry cannot assume kernel GS or a completed syscall stack switch. //! CR4.MCE remains off; machine checks shut down the CPU. +use core::sync::atomic::{AtomicU64, Ordering}; use litebox_common_linux::PtRegs; use spin::Once; use x86_64::{VirtAddr, structures::idt::InterruptDescriptorTable}; @@ -27,67 +28,87 @@ unsafe extern "C" { fn isr_alignment_check(); fn isr_simd_floating_point(); fn isr_ignore(); - fn isr_unexpected_stubs(); + /// One stub per vector from 32, each entering `isr_external`. + fn isr_external_stubs(); } -/// Stride of the per-vector stubs in `isr_unexpected_stubs`. +/// Stride of the per-vector stubs in `isr_external_stubs`. const UNEXPECTED_STUB_SIZE: usize = 8; const DOUBLE_FAULT_IST_INDEX: u16 = 0; const NMI_IST_INDEX: u16 = 1; +/// The IDT, and the state of the external interrupts it dispatches. +struct Interrupts { + idt: InterruptDescriptorTable, + /// See [`crate::set_interrupt_handler`]. + handler: Once bool>, + /// Bit `v % 64` of word `v / 64`: vector `v` was handled since it was + /// last taken ([`take_pending`]). + pending: [AtomicU64; 4], +} + +static INTERRUPTS: Once = Once::new(); + fn idt(ignored_vectors: &[u8]) -> &'static InterruptDescriptorTable { - static IDT_ONCE: Once = Once::new(); - IDT_ONCE.call_once(|| { - let mut idt = InterruptDescriptorTable::new(); - - // Safety: the stubs follow the interrupt calling convention. - unsafe { - idt.divide_error - .set_handler_addr(VirtAddr::from_ptr(isr_divide_error as *const ())); - idt.debug - .set_handler_addr(VirtAddr::from_ptr(isr_debug as *const ())); - idt.breakpoint - .set_handler_addr(VirtAddr::from_ptr(isr_breakpoint as *const ())); - idt.overflow - .set_handler_addr(VirtAddr::from_ptr(isr_overflow as *const ())); - idt.bound_range_exceeded - .set_handler_addr(VirtAddr::from_ptr(isr_bound_range_exceeded as *const ())); - idt.invalid_opcode - .set_handler_addr(VirtAddr::from_ptr(isr_invalid_opcode as *const ())); - idt.device_not_available - .set_handler_addr(VirtAddr::from_ptr(isr_device_not_available as *const ())); - for vector in 32..=u8::MAX { - let stub = isr_unexpected_stubs as *const () as usize - + usize::from(vector - 32) * UNEXPECTED_STUB_SIZE; - idt[vector].set_handler_addr(VirtAddr::new(stub as u64)); - } - idt.non_maskable_interrupt - .set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())) - .set_stack_index(NMI_IST_INDEX); - idt.double_fault - .set_handler_addr(VirtAddr::from_ptr(isr_double_fault as *const ())) - .set_stack_index(DOUBLE_FAULT_IST_INDEX); - idt.stack_segment_fault - .set_handler_addr(VirtAddr::from_ptr(isr_stack_segment_fault as *const ())); - idt.general_protection_fault - .set_handler_addr(VirtAddr::from_ptr( - isr_general_protection_fault as *const (), - )); - idt.page_fault - .set_handler_addr(VirtAddr::from_ptr(isr_page_fault as *const ())); - idt.x87_floating_point - .set_handler_addr(VirtAddr::from_ptr(isr_x87_floating_point as *const ())); - idt.alignment_check - .set_handler_addr(VirtAddr::from_ptr(isr_alignment_check as *const ())); - idt.simd_floating_point - .set_handler_addr(VirtAddr::from_ptr(isr_simd_floating_point as *const ())); - for &vector in ignored_vectors { - idt[vector].set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())); - } + &INTERRUPTS + .call_once(|| Interrupts { + idt: build_idt(ignored_vectors), + handler: Once::new(), + pending: [const { AtomicU64::new(0) }; 4], + }) + .idt +} + +fn build_idt(ignored_vectors: &[u8]) -> InterruptDescriptorTable { + let mut idt = InterruptDescriptorTable::new(); + + // Safety: the stubs follow the interrupt calling convention. + unsafe { + idt.divide_error + .set_handler_addr(VirtAddr::from_ptr(isr_divide_error as *const ())); + idt.debug + .set_handler_addr(VirtAddr::from_ptr(isr_debug as *const ())); + idt.breakpoint + .set_handler_addr(VirtAddr::from_ptr(isr_breakpoint as *const ())); + idt.overflow + .set_handler_addr(VirtAddr::from_ptr(isr_overflow as *const ())); + idt.bound_range_exceeded + .set_handler_addr(VirtAddr::from_ptr(isr_bound_range_exceeded as *const ())); + idt.invalid_opcode + .set_handler_addr(VirtAddr::from_ptr(isr_invalid_opcode as *const ())); + idt.device_not_available + .set_handler_addr(VirtAddr::from_ptr(isr_device_not_available as *const ())); + for vector in 32..=u8::MAX { + let stub = isr_external_stubs as *const () as usize + + usize::from(vector - 32) * UNEXPECTED_STUB_SIZE; + idt[vector].set_handler_addr(VirtAddr::new(stub as u64)); } - idt - }) + idt.non_maskable_interrupt + .set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())) + .set_stack_index(NMI_IST_INDEX); + idt.double_fault + .set_handler_addr(VirtAddr::from_ptr(isr_double_fault as *const ())) + .set_stack_index(DOUBLE_FAULT_IST_INDEX); + idt.stack_segment_fault + .set_handler_addr(VirtAddr::from_ptr(isr_stack_segment_fault as *const ())); + idt.general_protection_fault + .set_handler_addr(VirtAddr::from_ptr( + isr_general_protection_fault as *const (), + )); + idt.page_fault + .set_handler_addr(VirtAddr::from_ptr(isr_page_fault as *const ())); + idt.x87_floating_point + .set_handler_addr(VirtAddr::from_ptr(isr_x87_floating_point as *const ())); + idt.alignment_check + .set_handler_addr(VirtAddr::from_ptr(isr_alignment_check as *const ())); + idt.simd_floating_point + .set_handler_addr(VirtAddr::from_ptr(isr_simd_floating_point as *const ())); + for &vector in ignored_vectors { + idt[vector].set_handler_addr(VirtAddr::from_ptr(isr_ignore as *const ())); + } + } + idt } /// Requires [`super::gdt::init`]. The first call fixes `ignored_vectors`; @@ -104,8 +125,40 @@ pub fn init_idt(ignored_vectors: &[u8]) { idt(ignored_vectors).load(); } +/// # Panics +/// +/// Before [`init_idt`], or if a handler is already set. +pub(crate) fn set_external_interrupt_handler(handler: fn(u8) -> bool) { + let interrupts = INTERRUPTS.get().expect("the IDT is initialized at boot"); + let mut installed = false; + interrupts.handler.call_once(|| { + installed = true; + handler + }); + assert!(installed, "an interrupt handler is already set"); +} + +/// The vectors handled since the last call, as a 256-bit set. +pub(crate) fn take_pending() -> [u64; 4] { + INTERRUPTS.get().map_or([0; 4], |interrupts| { + core::array::from_fn(|word| interrupts.pending[word].swap(0, Ordering::Relaxed)) + }) +} + +/// Runs with interrupts disabled, on whatever stack and GS the interrupted +/// context had (see `isr_external`). #[unsafe(no_mangle)] -extern "C" fn unexpected_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) -> ! { +extern "C" fn external_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) { + if let (Some(interrupts), Ok(vector)) = (INTERRUPTS.get(), u8::try_from(vector)) + && interrupts + .handler + .get() + .is_some_and(|handler| handler(vector)) + { + interrupts.pending[usize::from(vector / 64)] + .fetch_or(1 << (vector % 64), Ordering::Relaxed); + return; + } panic!("EXCEPTION: UNEXPECTED INTERRUPT (vector {vector:#x}, RIP {rip:#x}, CS {cs:#x})"); } diff --git a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs index 16fa733d62..b494607295 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs @@ -436,6 +436,45 @@ impl X64PageTable<'_, M, ALIGN> { Ok(()) } + /// Maps device registers `frames` uncacheable at `start`, in the kernel + /// tables that task tables share. Pages already mapped to the same frame + /// are kept. + /// + /// # Safety + /// + /// The frames must be device registers, not RAM, and `start..` kernel + /// address space in an existing kernel PML4 slot, used for nothing else. + pub(crate) unsafe fn map_kernel_mmio( + &self, + start: Page, + frames: PhysFrameRange, + ) -> Result<(), MapToError> { + let mut allocator = PageTableAllocator::::new(); + let table_flags = PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::ACCESSED + | PageTableFlags::DIRTY; + // PCD and PWT select UC with the reset PAT. + let flags = PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::NO_EXECUTE + | PageTableFlags::NO_CACHE + | PageTableFlags::WRITE_THROUGH; + let mut inner = self.inner.lock(); + for (page, frame) in Page::range(start, start + frames.count() as u64).zip(frames) { + // Safety: device registers (the caller's contract), mapped only here. + match unsafe { + inner.map_to_with_table_flags(page, frame, flags, table_flags, &mut allocator) + } { + // A newly present entry needs no flush. + Ok(flush) => flush.ignore(), + Err(MapToError::PageAlreadyMapped(mapped)) if mapped == frame => {} + Err(error) => return Err(error), + } + } + Ok(()) + } + /// # Panics /// /// Panics if the frame allocation fails. diff --git a/litebox_platform_vm_kernel/src/dma.rs b/litebox_platform_vm_kernel/src/dma.rs new file mode 100644 index 0000000000..d0aacd90b2 --- /dev/null +++ b/litebox_platform_vm_kernel/src/dma.rs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! [`litebox_hal::dma::Hal`]: DMA from the page allocator (RAM is +//! direct-mapped, so contiguous), and device memory mapped uncacheable in +//! the kernel's PML4 slot, whose tables task page tables share, so they see +//! later mappings. Device memory below [`MMIO_WINDOW`] is mapped like RAM, +//! at `PA + KERNEL_OFFSET`; above it (firmware may place 64-bit BARs far +//! above RAM), in the window. + +use crate::mm::MemoryProvider; +use crate::{KERNEL_OFFSET, VmKernel}; +use core::ops::Range; +use core::ptr::NonNull; +use litebox_hal::dma::{Hal, Mmio}; +use x86_64::structures::paging::{Page, PageSize, PhysFrame, Size4KiB}; +use x86_64::{PhysAddr, VirtAddr}; + +/// The top of the kernel's PML4 slot, as offsets from `KERNEL_OFFSET`. RAM +/// must end below it. +pub(crate) const MMIO_WINDOW: Range = 448 << 30..512 << 30; + +fn order(pages: usize) -> u32 { + pages.next_power_of_two().trailing_zeros() +} + +// Safety: page-allocator blocks are contiguous, exclusively owned until +// freed, and mapped writable at `PA + KERNEL_OFFSET`. Device memory is +// mapped uncacheable for the kernel's lifetime, and RAM is refused. +unsafe impl Hal for VmKernel { + fn dma_alloc(&self, pages: usize) -> Option<(NonNull, u64)> { + let va = NonNull::new(Self::mem_allocate_pages(order(pages))?)?; + Some((va, Self::va_to_pa(VirtAddr::from_ptr(va.as_ptr())).as_u64())) + } + + unsafe fn dma_dealloc(&self, va: NonNull, pages: usize) { + // Safety: from `dma_alloc` with the same `pages`. + unsafe { Self::mem_free_pages(va.as_ptr(), order(pages)) }; + } + + unsafe fn map_mmio(&self, pa: u64, len: usize) -> Option { + let end = pa + .checked_add(u64::try_from(len).ok()?) + .filter(|_| len != 0)?; + let first = PhysFrame::::containing_address(PhysAddr::try_new(pa).ok()?); + let last = + PhysFrame::containing_address(PhysAddr::try_new(end).ok()?.align_up(Size4KiB::SIZE)); + let ram = || self.ram_frame_ranges.iter(); + if ram().any(|r| r.start < last && first < r.end) + || ram().any(|r| r.end.start_address().as_u64() > MMIO_WINDOW.start) + { + return None; + } + let start = if last.start_address().as_u64() <= MMIO_WINDOW.start { + first.start_address().as_u64() + } else { + let mut next = self.mmio_next.lock(); + let start = *next; + let after = start.checked_add(last.start_address() - first.start_address())?; + if after > MMIO_WINDOW.end { + return None; + } + *next = after; + start + }; + let page = Page::from_start_address(VirtAddr::new(KERNEL_OFFSET + start)).ok()?; + // Safety: device registers (the caller's contract), not RAM (checked), + // at their direct-map address or a part of the window only this + // mapping gets. + unsafe { + self.page_table_manager + .base_page_table + .map_kernel_mmio(page, PhysFrame::range(first, last)) + } + .ok()?; + let va = page.start_address() + (pa - first.start_address().as_u64()); + // Safety: mapped uncacheable above, for the kernel's lifetime. + Some(unsafe { Mmio::new(NonNull::new(va.as_mut_ptr())?, len) }) + } +} diff --git a/litebox_platform_vm_kernel/src/lib.rs b/litebox_platform_vm_kernel/src/lib.rs index 04e4dc88ce..63fd2c0f84 100644 --- a/litebox_platform_vm_kernel/src/lib.rs +++ b/litebox_platform_vm_kernel/src/lib.rs @@ -12,8 +12,15 @@ //! //! Assumptions: a single CPU, no scheduler, and not a confidential VM (no //! #VE/#VC/#HV). -//! There is no timer, so user code that never enters the kernel is never -//! preempted. +//! +//! Devices: [`VmKernel`] is the drivers' [`litebox_hal::dma::Hal`]. +//! +//! Interrupts: the kernel runs with interrupts disabled, except while it +//! halts ([`VmKernel::halt_until_interrupt`]); user code runs with them +//! enabled. External interrupts go to the handler the runner installs +//! ([`set_interrupt_handler`]), which returns to the interrupted code: there +//! is no preemption, so user code that never enters the kernel keeps the +//! CPU. #![cfg(target_arch = "x86_64")] #![no_std] @@ -67,6 +74,7 @@ extern crate alloc; mod arch; mod boot; pub mod clock; +mod dma; pub mod mm; mod per_cpu; pub mod providers; @@ -89,6 +97,37 @@ pub struct BootConfig<'a> { pub ignored_vectors: &'a [u8], } +/// Installs the handler for external interrupts (vectors 32 and up, except +/// [`BootConfig::ignored_vectors`]), which returns whether it handled the +/// vector; an unhandled one panics. Before this, every external interrupt +/// panics. +/// +/// The handler runs with interrupts disabled, on the interrupted context's GS +/// (user mode, or [`VmKernel::halt_until_interrupt`]): it must not touch +/// per-CPU data, block, or take locks, only quiet the source. The kernel acts +/// on handled vectors later ([`VmKernel::take_pending_interrupts`]). +/// +/// # Panics +/// +/// On a second call. +pub fn set_interrupt_handler(handler: fn(u8) -> bool) { + arch::interrupts::set_external_interrupt_handler(handler); +} + +/// A set of interrupt vectors. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct InterruptVectors([u64; 4]); + +impl InterruptVectors { + pub fn contains(&self, vector: u8) -> bool { + self.0[usize::from(vector / 64)] & (1 << (vector % 64)) != 0 + } + + pub fn iter(&self) -> impl Iterator + '_ { + (0..=u8::MAX).filter(|&vector| self.contains(vector)) + } +} + /// Valid until unregistered; its representation is private to the platform. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct AddressSpaceId(usize); @@ -281,6 +320,8 @@ pub struct VmKernel { clock: &'static dyn clock::ClockSource, /// Guest RAM owned by this kernel (mapped at `PA + KERNEL_OFFSET`). ram_frame_ranges: alloc::vec::Vec>, + /// The next free offset in `dma::MMIO_WINDOW`. + mmio_next: spin::Mutex, } /// Confines user pointers to the user range and opens SMAP around accesses. @@ -430,9 +471,24 @@ impl VmKernel { page_table_manager: PageTableManager::new(base_pt), clock, ram_frame_ranges, + mmio_next: spin::Mutex::new(dma::MMIO_WINDOW.start), })) } + /// The vectors the interrupt handler handled since the last call. + pub fn take_pending_interrupts(&self) -> InterruptVectors { + InterruptVectors(arch::interrupts::take_pending()) + } + + /// Halts until an interrupt is handled, with interrupts enabled only for + /// the halt. An interrupt that became pending while they were disabled + /// ends the halt at once, so check for work first and halt after: none + /// is missed. + pub fn halt_until_interrupt(&self) { + x86_64::instructions::interrupts::enable_and_hlt(); + x86_64::instructions::interrupts::disable(); + } + /// Only valid for addresses in the kernel's direct mapping. pub fn va_to_pa(va: VirtAddr) -> PhysAddr { ::va_to_pa(va) diff --git a/litebox_runner_optee_on_vm_userland/Cargo.toml b/litebox_runner_optee_on_vm_userland/Cargo.toml index df94def7d4..beb44fa899 100644 --- a/litebox_runner_optee_on_vm_userland/Cargo.toml +++ b/litebox_runner_optee_on_vm_userland/Cargo.toml @@ -6,6 +6,7 @@ description = "Runs an OP-TEE TA in ring 3 on top of the LiteBox VM kernel" [dependencies] litebox = { version = "0.1.0", path = "../litebox" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_common_linux = { version = "0.1.0", path = "../litebox_common_linux" } litebox_common_optee = { version = "0.1.0", path = "../litebox_common_optee" } litebox_broker_local_vm_userland = { version = "0.1.0", path = "../litebox_broker_local_vm_userland" } diff --git a/litebox_runner_optee_on_vm_userland/src/optee.rs b/litebox_runner_optee_on_vm_userland/src/optee.rs index 5f98bdbe6d..9b2d8fc972 100644 --- a/litebox_runner_optee_on_vm_userland/src/optee.rs +++ b/litebox_runner_optee_on_vm_userland/src/optee.rs @@ -15,6 +15,7 @@ use core::cell::RefCell; use core::ops::Range; use litebox::platform::RawConstPointer as _; use litebox::utils::TruncateExt as _; +use litebox_broker_protocol::fs::{FileAccessMode, FileMode, FileOpenFlags}; use litebox_common_linux::PtRegs; use litebox_common_optee::envelope::{ LDELF_IMAGE, OPEN_SESSION_META_PARAMS, TA_IMAGE, parse_open_session_meta, rmem_buffer, @@ -132,14 +133,28 @@ pub fn serve(info: &StartupInfo) -> ! { let platform: &'static Platform = Box::leak(Box::new(VmUserland::new(info))); let session_manager: &'static SessionManager = Box::leak(Box::new(SessionManager::new())); - let local = litebox_broker_local_vm_userland::connect(info) + let (local, mut notifications) = litebox_broker_local_vm_userland::connect(info) .unwrap_or_else(|e| panic!("broker association: {e:?}")); kcall::restrict(CallSet::ALL.without(CallId::BrokerHandshake), ProtSet::ALL) .unwrap_or_else(|status| panic!("lockdown: {status:?}")); let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let shim = - litebox_shim_optee::OpteeShimBuilder::new_with_litebox(platform, session_manager, litebox) - .build(); + let mut builder = + litebox_shim_optee::OpteeShimBuilder::new_with_litebox(platform, session_manager, litebox); + // TA and ldelf output goes to the broker's standard output. + match builder.litebox().open_file( + &litebox::fs::Context::new(), + "/dev/stdout", + FileAccessMode::WriteOnly, + FileOpenFlags::NONE, + FileMode::empty(), + ) { + Ok(stdout) => builder = builder.with_log_file(stdout), + Err(error) => log::warn!("TA output stays in the log: /dev/stdout: {error:?}"), + } + if cfg!(debug_assertions) { + check_stdin(builder.litebox(), &mut notifications, info.tsc_khz); + } + let shim = builder.build(); // Safety: mapped read-only for the process's lifetime; nothing writes // them. @@ -171,7 +186,7 @@ pub fn serve(info: &StartupInfo) -> ! { // Requires upcalls, registered by `ready`. check_guest_memory_access(); check_lockdown_validation(); - litebox_broker_local_vm_userland::self_check(); + litebox_broker_local_vm_userland::self_check(¬ifications, info.tsc_khz); } loop { let reply = runner.serve(request); @@ -729,6 +744,64 @@ fn check_guest_memory_access() { assert_eq!(read(), None, "read from unmapped guest memory"); } +/// Self-check (debug builds): standard input through the broker, which TAs +/// never read. Waits briefly for input (on broker notifications), and echoes +/// what arrived to standard output. +fn check_stdin( + litebox: &litebox::LiteBox, + notifications: &mut litebox_broker_local_vm_userland::Notifications, + tsc_khz: u64, +) { + use litebox::fs::errors::ReadError; + const WAIT_MS: u64 = 20; + + let context = litebox::fs::Context::new(); + let open = |path, access| { + litebox + .open_file( + &context, + path, + access, + FileOpenFlags::NONBLOCKING, + FileMode::empty(), + ) + .unwrap_or_else(|error| panic!("opening {path}: {error:?}")) + }; + let stdin = open("/dev/stdin", FileAccessMode::ReadOnly); + // Safety: RDTSC has no side effects; the kernel allows it in ring 3. + let deadline = unsafe { core::arch::x86_64::_rdtsc() } + WAIT_MS * tsc_khz; + let mut input = [0u8; 256]; + let len = loop { + match litebox.read_file(&stdin, &mut input, None, None) { + Ok(len) => break len, + Err(ReadError::WouldBlock) => { + let dispatch = litebox.broker_notification_dispatcher(); + match notifications.receive(deadline, dispatch) { + Ok(0) => break 0, + Ok(_) => {} + Err(error) => panic!("waiting for standard input: {error}"), + } + } + Err(error) => panic!("reading standard input: {error:?}"), + } + }; + litebox.close_file(&stdin).expect("closing standard input"); + if len == 0 { + log::debug!("stdin check: no input within {WAIT_MS} ms"); + return; + } + let stdout = open("/dev/stdout", FileAccessMode::WriteOnly); + for bytes in [&b"[stdin] "[..], &input[..len]] { + litebox + .write_file(&stdout, bytes, None, None) + .expect("writing standard output"); + } + litebox + .close_file(&stdout) + .expect("closing standard output"); + log::info!("stdin check: echoed {len} bytes"); +} + /// Self-check (debug builds): unknown lockdown bits are rejected. fn check_lockdown_validation() { use litebox_common_vm_abi::{RestrictRequest, Status}; diff --git a/litebox_runner_vm_kernel/Cargo.toml b/litebox_runner_vm_kernel/Cargo.toml index dbe6f2ea8b..9e956bbc92 100644 --- a/litebox_runner_vm_kernel/Cargo.toml +++ b/litebox_runner_vm_kernel/Cargo.toml @@ -8,6 +8,8 @@ description = "LiteBox as the guest kernel of a QEMU VM, hosting ring-3 runner p arrayvec = { version = "0.7.6", default-features = false } base64 = { version = "0.22.1", default-features = false, features = ["alloc"] } litebox_bootloader = { version = "0.1.0", path = "../litebox_bootloader" } +litebox_broker_core = { version = "0.1.0", path = "../litebox_broker_core" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_vm_kernel = { version = "0.1.0", path = "../litebox_broker_vm_kernel" } litebox_hal = { version = "0.1.0", path = "../litebox_hal" } litebox_common_optee = { version = "0.1.0", path = "../litebox_common_optee" } @@ -16,6 +18,7 @@ litebox_platform_vm_kernel = { version = "0.1.0", path = "../litebox_platform_vm litebox_shim_vm_kernel = { version = "0.1.0", path = "../litebox_shim_vm_kernel" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } log = { version = "0.4", default-features = false } +spin = { version = "0.10.0", default-features = false, features = ["spin_mutex"] } serde = { version = "1.0.142", default-features = false, features = ["derive", "alloc"] } serde_json = { version = "1.0.142", default-features = false, features = ["alloc"] } tar-no-std = { version = "0.3.3", default-features = false, features = ["alloc"] } diff --git a/litebox_runner_vm_kernel/src/devices.rs b/litebox_runner_vm_kernel/src/devices.rs new file mode 100644 index 0000000000..b8217af896 --- /dev/null +++ b/litebox_runner_vm_kernel/src/devices.rs @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Devices the kernel drives, from `litebox_hal`, for the broker +//! ([`litebox_broker_vm_kernel::Config`]): +//! +//! - Runner processes' standard streams: a virtio console if there is one, +//! else output to the serial console and no input. +//! - Events: the console's interrupt, and the PIT for deadlines. +//! +//! The interrupt handler only masks the IRQ (the platform records the +//! vector); the kernel services the device and unmasks it when it next polls +//! or halts. + +use alloc::boxed::Box; +use core::sync::atomic::{AtomicBool, Ordering}; +use litebox_broker_core::readiness::{ReadinessRegistration, ReadinessWatchers}; +use litebox_broker_core::stdio::{ + StdioOutputStream, StdioProvider, StdioProviderError, StdioStream, +}; +use litebox_broker_protocol::readiness::ReadinessFlags; +use litebox_hal::virtio::console::VirtioConsole; +use litebox_hal::{interrupt, timer}; +use litebox_platform_vm_kernel::VmKernel; +use spin::Mutex; + +/// How long a console write may wait for the device to take output before +/// the rest is dropped. +const WRITE_TIMEOUT_NANOS: u64 = 100_000_000; + +/// See `litebox_platform_vm_kernel::set_interrupt_handler`: no locks, no +/// per-CPU data. The IRQ stays masked until [`Devices::acknowledge`]. +fn on_interrupt(vector: u8) -> bool { + let Some(irq) = interrupt::irq(vector) else { + return false; + }; + interrupt::mask_and_acknowledge(irq); + true +} + +struct Console { + device: Mutex, + irq: Option, + /// Standard-input files, to wake when input arrives. + stdin_watchers: Mutex, + /// Whether input was pending when last looked at; watchers are woken + /// when it starts to be. + stdin_readable: AtomicBool, +} + +impl Console { + fn readable(&self) -> bool { + self.device.lock().has_input() + } +} + +pub struct Devices { + platform: &'static VmKernel, + tsc_khz: u64, + console: Option, +} + +impl Devices { + /// Brings up the devices and enables their interrupts. Call once. + pub fn init(platform: &'static VmKernel, tsc_khz: u64) -> &'static Self { + litebox_platform_vm_kernel::set_interrupt_handler(on_interrupt); + timer::disarm(); + interrupt::unmask(timer::IRQ); + let console = match VirtioConsole::probe(platform) { + None => { + litebox_util_log::info!( + "no virtio console: standard output goes to the serial console" + ); + None + } + Some(Err(error)) => { + litebox_util_log::warn!( + error:% = error; + "virtio console unusable: standard output goes to the serial console" + ); + None + } + Some(Ok(device)) => { + let irq = device.transport().irq(); + litebox_util_log::info!( + pci:% = device.transport().function(), + irq:? = irq; + "virtio console: standard streams" + ); + if let Some(irq) = irq { + interrupt::unmask(irq); + } + Some(Console { + device: Mutex::new(device), + irq, + stdin_watchers: Mutex::new(ReadinessWatchers::default()), + stdin_readable: AtomicBool::new(false), + }) + } + }; + Box::leak(Box::new(Self { + platform, + tsc_khz, + console, + })) + } + + /// Acknowledges recorded interrupts at their devices and unmasks them. + fn acknowledge(&self) { + let pending = self.platform.take_pending_interrupts(); + if let Some(console) = &self.console + && let Some(irq) = console.irq + && pending.contains(interrupt::vector(irq)) + { + // Before unmasking, or the line would still be asserted. + console.device.lock().transport().acknowledge_interrupt(); + } + for irq in pending.iter().filter_map(interrupt::irq) { + interrupt::unmask(irq); + } + } + + /// Halts until an interrupt, or until `deadline` (TSC). + fn halt(&self, deadline: Option) { + if let Some(deadline) = deadline { + let now = rdtsc(); + if now >= deadline { + return; + } + let nanos = u128::from(deadline - now) * 1_000_000 / u128::from(self.tsc_khz); + timer::arm_oneshot(u64::try_from(nanos).unwrap_or(u64::MAX)); + } + self.platform.halt_until_interrupt(); + if deadline.is_some() { + timer::disarm(); + } + self.acknowledge(); + } +} + +fn rdtsc() -> u64 { + // Safety: RDTSC has no side effects. + unsafe { core::arch::x86_64::_rdtsc() } +} + +impl litebox_broker_vm_kernel::Events for Devices { + fn poll(&self) { + self.acknowledge(); + // Wake watchers when input starts to be pending. + if let Some(console) = &self.console { + let readable = console.readable(); + if readable && !console.stdin_readable.swap(true, Ordering::Relaxed) { + console.stdin_watchers.lock().publish(ReadinessFlags::READ); + } else if !readable { + console.stdin_readable.store(false, Ordering::Relaxed); + } + } + } + + fn wait(&self, deadline: Option) -> bool { + let interrupts = self + .console + .as_ref() + .is_some_and(|console| console.irq.is_some()); + if deadline.is_none() && !interrupts { + return false; + } + self.halt(deadline); + self.poll(); + true + } +} + +/// Standard streams over [`Devices`]. Never a terminal. Without a console, +/// standard input is at its end. +pub struct Stdio(pub &'static Devices); + +impl StdioProvider for Stdio { + fn read(&self, output: &mut [u8]) -> Result { + let Some(console) = &self.0.console else { + return Ok(0); + }; + match console.device.lock().read(output) { + 0 => Err(StdioProviderError::WouldBlock), + read => Ok(read), + } + } + + /// Waits (bounded) for the console to take all of `input`. + fn write(&self, _stream: StdioOutputStream, input: &[u8]) -> Result { + let Some(console) = &self.0.console else { + litebox_hal::console::print_bytes(input); + return Ok(input.len()); + }; + let deadline = rdtsc() + WRITE_TIMEOUT_NANOS * self.0.tsc_khz / 1_000_000; + let mut written = 0; + loop { + written += console.device.lock().write(&input[written..]); + if written == input.len() || rdtsc() >= deadline { + break; + } + // Until the device returns a transmit buffer. + self.0.halt(Some(deadline)); + } + if written == 0 { + return Err(StdioProviderError::WouldBlock); + } + Ok(written) + } + + fn is_terminal(&self, _stream: StdioStream) -> bool { + false + } + + fn readiness(&self, stream: StdioStream) -> ReadinessFlags { + match stream { + StdioStream::Stdin => match &self.0.console { + Some(console) if !console.readable() => ReadinessFlags(0), + _ => ReadinessFlags::READ, + }, + StdioStream::Stdout | StdioStream::Stderr => ReadinessFlags::WRITE, + } + } + + fn watch( + &self, + stream: StdioStream, + registration: &ReadinessRegistration, + ) -> litebox_broker_core::Result<()> { + if let (StdioStream::Stdin, Some(console)) = (stream, &self.0.console) { + console.stdin_watchers.lock().watch(registration)?; + if console.readable() { + registration.publish(ReadinessFlags::READ)?; + } + } + Ok(()) + } +} diff --git a/litebox_runner_vm_kernel/src/main.rs b/litebox_runner_vm_kernel/src/main.rs index 4b46a44556..b11eeda1a0 100644 --- a/litebox_runner_vm_kernel/src/main.rs +++ b/litebox_runner_vm_kernel/src/main.rs @@ -6,7 +6,8 @@ //! `litebox_runner_optee_on_vm_userland` processes, one per TA instance //! ([`service::optee`]), //! driven by a test client ([`client::script`]). QEMU exits with 33 if all -//! tests pass, 65 otherwise. +//! tests pass, 65 otherwise. Runner processes' standard streams are a virtio +//! console, if there is one ([`devices`]). #![cfg(target_arch = "x86_64")] #![no_std] @@ -16,6 +17,7 @@ extern crate alloc; mod client; +mod devices; mod payload; mod service; @@ -30,7 +32,12 @@ fn kernel_main(kernel: Kernel) -> ! { set_platform_root_key(&litebox_hal::prk::development()); let payload = payload::Payload::read(kernel.boot_info).unwrap_or_else(|e| panic!("payload: {e}")); - let mut service = service::optee::Optee::new(kernel.platform, kernel.tsc_khz, &payload); + let devices = devices::Devices::init(kernel.platform, kernel.tsc_khz); + let broker = litebox_broker_vm_kernel::Broker::new(litebox_broker_vm_kernel::Config { + stdio: alloc::sync::Arc::new(devices::Stdio(devices)), + events: devices, + }); + let mut service = service::optee::Optee::new(kernel.platform, kernel.tsc_khz, &payload, broker); client::script::run(&payload, &mut service); console_println!("[litebox] ALL TESTS PASSED"); power::exit(true) diff --git a/litebox_runner_vm_kernel/src/service/optee.rs b/litebox_runner_vm_kernel/src/service/optee.rs index c3dd7740bd..ee04a48275 100644 --- a/litebox_runner_vm_kernel/src/service/optee.rs +++ b/litebox_runner_vm_kernel/src/service/optee.rs @@ -36,7 +36,12 @@ impl Optee { /// # Panics /// /// On a missing or malformed payload file. - pub fn new(platform: &'static VmKernel, tsc_khz: u64, payload: &Payload) -> Self { + pub fn new( + platform: &'static VmKernel, + tsc_khz: u64, + payload: &Payload, + broker: litebox_broker_vm_kernel::Broker, + ) -> Self { let file = |name| { payload .file(name) @@ -66,7 +71,6 @@ impl Optee { policies.push((uuid, policy)); } assert!(!tas.is_empty(), "payload has no tas/.elf"); - let broker = litebox_broker_vm_kernel::Broker::new(); let spawn: Spawn = Box::new(move |uuid: &TeeUuid| { let Some(ta) = tas.get(uuid) else { litebox_util_log::error!(uuid:? = uuid; "no such TA"); diff --git a/litebox_shim_optee/Cargo.toml b/litebox_shim_optee/Cargo.toml index c3f9325c31..25ef836166 100644 --- a/litebox_shim_optee/Cargo.toml +++ b/litebox_shim_optee/Cargo.toml @@ -9,6 +9,7 @@ ctr = { version = "0.8", default-features = false } elf = { version = "0.8.0", default-features = false } hashbrown = "0.15.2" litebox = { path = "../litebox/", version = "0.1.0" } +litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } litebox_common_linux = { path = "../litebox_common_linux/", version = "0.1.0" } litebox_common_optee = { path = "../litebox_common_optee/", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } diff --git a/litebox_shim_optee/src/lib.rs b/litebox_shim_optee/src/lib.rs index bfc98591be..da110e4953 100644 --- a/litebox_shim_optee/src/lib.rs +++ b/litebox_shim_optee/src/lib.rs @@ -185,6 +185,7 @@ pub struct OpteeShimBuilder { session_manager: &'static session::SessionManager, litebox: LiteBox, ta_signing_cert: &'static [u8], + log_file: Option, } impl OpteeShimBuilder { @@ -209,6 +210,7 @@ impl OpteeShimBuilder { session_manager, litebox, ta_signing_cert: &[], + log_file: None, } } @@ -219,6 +221,15 @@ impl OpteeShimBuilder { self } + /// Where TA and ldelf messages (`utee_log`) go, as written, instead of + /// the log: a file of [`Self::litebox`], such as the broker's + /// `/dev/stdout`. Messages the file does not take are logged. + #[must_use] + pub fn with_log_file(mut self, log_file: litebox::fs::FileFd) -> Self { + self.log_file = Some(log_file); + self + } + /// Returns the litebox object for the shim. pub fn litebox(&self) -> &LiteBox { &self.litebox @@ -236,6 +247,7 @@ impl OpteeShimBuilder { dynamic_ta_uuid_map: TaUuidMap::new(), ta_signing_cert: self.ta_signing_cert, pta_busy: spin::mutex::SpinMutex::new(HashSet::new()), + log_file: self.log_file, page_table_keepalive: None, }); OpteeShim(global) @@ -269,6 +281,8 @@ struct GlobalState { /// blocking/queuing the caller until the PTA is free. We currently reject /// instead of serialize; revisit if a PTA needs true serialization. pta_busy: spin::mutex::SpinMutex>, + /// See [`OpteeShimBuilder::with_log_file`]. + log_file: Option, /// Keeps the TA page table alive; declared last to drop after all other shim state. // TODO: Replace type erasure with a typed platform page-table handle. page_table_keepalive: Option>, diff --git a/litebox_shim_optee/src/syscalls/tee.rs b/litebox_shim_optee/src/syscalls/tee.rs index 363c27e1f5..8f2910795a 100644 --- a/litebox_shim_optee/src/syscalls/tee.rs +++ b/litebox_shim_optee/src/syscalls/tee.rs @@ -58,12 +58,26 @@ impl Task { litebox_common_optee::TeeResult::TargetDead as usize } - /// A system call to print out a message. - #[expect( - clippy::unused_self, - reason = "self was used by the old platform-threaded logging API" - )] + /// A system call to print out a message: to the log file, as is (OP-TEE + /// traces end their lines, and `printf` may log fragments), else to the + /// log. A message the file takes in part is not logged again. pub fn sys_log(&self, buf: &[u8]) -> Result<(), TeeResult> { + if let Some(file) = &self.global.log_file { + let mut written = 0; + while written < buf.len() { + match self + .global + .litebox + .write_file(file, &buf[written..], None, None) + { + Ok(n) if n != 0 => written += n, + _ => break, + } + } + if written != 0 || buf.is_empty() { + return Ok(()); + } + } let msg = core::str::from_utf8(buf).map_err(|_| TeeResult::BadFormat)?; litebox_util_log::info!(msg:% = msg; "sys_log"); Ok(()) From 442bddc97b9ad69f013914d880dc0bbd802741e5 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Fri, 9 Oct 2026 23:44:52 +0000 Subject: [PATCH 2/7] use local APIC --- litebox_hal/src/interrupt.rs | 226 ++++++++++++++---- litebox_hal/src/lib.rs | 2 - litebox_hal/src/pci.rs | 90 ++++++- litebox_hal/src/timer.rs | 39 --- litebox_hal/src/virtio/console.rs | 16 +- litebox_hal/src/virtio/mod.rs | 6 +- litebox_hal/src/virtio/pci.rs | 85 +++---- .../src/arch/x86/interrupts.rs | 19 +- litebox_platform_vm_kernel/src/lib.rs | 28 +-- litebox_runner_vm_kernel/src/devices.rs | 82 +++---- 10 files changed, 355 insertions(+), 238 deletions(-) delete mode 100644 litebox_hal/src/timer.rs diff --git a/litebox_hal/src/interrupt.rs b/litebox_hal/src/interrupt.rs index b86aa8b63b..c3b75a0e9a 100644 --- a/litebox_hal/src/interrupt.rs +++ b/litebox_hal/src/interrupt.rs @@ -1,13 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Interrupt controllers: the legacy 8259 PICs. -//! -//! Handlers mask their IRQ ([`mask_and_acknowledge`]) until the device is -//! serviced ([`unmask`]), as a level-triggered (PCI) line stays asserted -//! until then. Call with interrupts disabled: masks are read-modify-write. +//! Interrupt controllers: the local APIC, for device interrupts (MSI) and +//! the timer, in x2APIC mode (as on LVBS) where the CPU has it, else in +//! xAPIC mode (e.g., QEMU's TCG); the legacy 8259 PICs are only masked. +use crate::dma::{Hal, Mmio}; +use crate::pci::MsiMessage; use x86_64::instructions::port::Port; +use x86_64::registers::model_specific::Msr; const LEGACY_PIC_VECTOR_BASE: u8 = 0x20; @@ -15,28 +16,39 @@ const MASTER_CMD: u16 = 0x20; const MASTER_DATA: u16 = 0x21; const SLAVE_CMD: u16 = 0xa0; const SLAVE_DATA: u16 = 0xa1; -const CASCADE_IRQ: u8 = 2; -const EOI: u8 = 0x20; -/// The vector of PIC IRQ `irq` (0..16). -pub const fn vector(irq: u8) -> u8 { - LEGACY_PIC_VECTOR_BASE + irq -} +const IA32_APIC_BASE: u32 = 0x1b; +const IA32_APIC_BASE_EN: u64 = 1 << 11; +const IA32_APIC_BASE_EXTD: u64 = 1 << 10; +const IA32_TSC_DEADLINE: u32 = 0x6e0; -/// The PIC IRQ behind `vector`, if it is a PIC vector. -pub const fn irq(vector: u8) -> Option { - if vector >= LEGACY_PIC_VECTOR_BASE && vector < LEGACY_PIC_VECTOR_BASE + 16 { - Some(vector - LEGACY_PIC_VECTOR_BASE) - } else { - None - } +/// Local APIC registers, as xAPIC offsets; x2APIC MSR `0x800 + offset / 16`. +mod reg { + pub const ID: usize = 0x20; + pub const EOI: usize = 0xb0; + pub const SVR: usize = 0xf0; + pub const LVT_TIMER: usize = 0x320; + pub const TIMER_INITIAL_COUNT: usize = 0x380; + pub const TIMER_CURRENT_COUNT: usize = 0x390; + pub const TIMER_DIVIDE: usize = 0x3e0; } -/// PIC IRQ7/IRQ15 can arrive while masked; SeaBIOS leaves LAPIC SVR at 0xff. +const SVR_ENABLE: u32 = 1 << 8; +const LVT_TIMER_MASKED: u32 = 1 << 16; +const LVT_TIMER_TSC_DEADLINE: u32 = 0b10 << 17; +/// Divide the timer's clock by 16. +const TIMER_DIVIDE_BY_16: u32 = 0b0011; +const TIMER_CALIBRATION_MS: u64 = 10; + +/// The local APIC's spurious vector; needs no EOI. +const APIC_SPURIOUS_VECTOR: u8 = 0xff; + +/// PIC IRQ7/IRQ15 can arrive while masked, and the local APIC raises its +/// spurious vector. pub const SPURIOUS_VECTORS: [u8; 3] = [ LEGACY_PIC_VECTOR_BASE + 7, LEGACY_PIC_VECTOR_BASE + 15, - 0xff, + APIC_SPURIOUS_VECTOR, ]; /// Must precede user entry (IF=1). PVH's PIC vectors overlap CPU exceptions. @@ -59,42 +71,154 @@ pub fn init_legacy_pics() { } } -/// The mask register and bit of `irq`. -fn mask_register(irq: u8) -> (Port, u8) { - assert!(irq < 16, "PIC IRQs are 0..16"); - if irq < 8 { - (Port::new(MASTER_DATA), 1 << irq) - } else { - (Port::new(SLAVE_DATA), 1 << (irq - 8)) - } +/// This CPU's local APIC, enabled, with its timer raising a vector when +/// armed. +pub struct LocalApic { + /// The xAPIC registers; `None` in x2APIC mode. + xapic: Option, + /// Timer ticks per TSC cycle, as (ticks, cycles); `None` for the + /// TSC-deadline timer. + one_shot: Option<(u64, u64)>, } -/// Lets `irq` interrupt (when IF=1); an IRQ on the slave also needs the -/// cascade. -pub fn unmask(irq: u8) { - let (mut port, bit) = mask_register(irq); - // Safety: only the PICs' mask registers change; nothing else drives the - // PICs, and the caller excludes interleaving (see the module docs). - unsafe { - let mask = port.read(); - port.write(mask & !bit); +fn has_cpu_feature(ecx_bit: u32) -> bool { + core::arch::x86_64::__cpuid_count(1, 0).ecx & (1 << ecx_bit) != 0 +} + +fn rdtsc() -> u64 { + // Safety: RDTSC has no side effects. + unsafe { core::arch::x86_64::_rdtsc() } +} + +impl LocalApic { + /// Enables the local APIC and sets its timer to raise `timer_vector`, + /// disarmed. Calibrates the timer against the TSC (`tsc_khz`) if it has + /// no TSC-deadline mode. + /// + /// # Panics + /// + /// If the xAPIC registers cannot be mapped. + pub fn init(hal: &dyn Hal, timer_vector: u8, tsc_khz: u64) -> Self { + // The local APIC belongs to this kernel, and interrupts are disabled. + // The SDM requires enabling xAPIC before x2APIC. + let mut base = Msr::new(IA32_APIC_BASE); + // Safety: see above. + let value = unsafe { base.read() } | IA32_APIC_BASE_EN; + let xapic = if has_cpu_feature(21) { + // Safety: see above. + unsafe { + base.write(value); + base.write(value | IA32_APIC_BASE_EXTD); + } + None + } else { + // Safety: see above. + unsafe { base.write(value) }; + // Safety: the APIC base names the local APIC's registers. + let registers = unsafe { hal.map_mmio(value & 0xf_ffff_f000, 0x1000) }; + Some(registers.expect("the local APIC's registers can be mapped")) + }; + let mut apic = Self { + xapic, + one_shot: None, + }; + let svr = apic.read(reg::SVR); + apic.write(reg::SVR, svr | SVR_ENABLE | u32::from(APIC_SPURIOUS_VECTOR)); + if has_cpu_feature(24) { + apic.write( + reg::LVT_TIMER, + LVT_TIMER_TSC_DEADLINE | u32::from(timer_vector), + ); + } else { + apic.write(reg::TIMER_DIVIDE, TIMER_DIVIDE_BY_16); + apic.write(reg::LVT_TIMER, LVT_TIMER_MASKED | u32::from(timer_vector)); + apic.write(reg::TIMER_INITIAL_COUNT, u32::MAX); + let start = rdtsc(); + let cycles = TIMER_CALIBRATION_MS * tsc_khz; + while rdtsc() - start < cycles {} + let ticks = u64::from(u32::MAX - apic.read(reg::TIMER_CURRENT_COUNT)); + apic.write(reg::TIMER_INITIAL_COUNT, 0); + apic.write(reg::LVT_TIMER, u32::from(timer_vector)); + apic.one_shot = Some((ticks.max(1), cycles)); + } + litebox_util_log::info!( + mode:% = if apic.xapic.is_some() { "xAPIC" } else { "x2APIC" }, + timer:% = if apic.one_shot.is_some() { "one-shot" } else { "TSC-deadline" }; + "local APIC" + ); + apic } - if irq >= 8 { - unmask(CASCADE_IRQ); + + /// x2APIC register `register`'s MSR. + fn msr(register: usize) -> Msr { + Msr::new(0x800 + u32::try_from(register >> 4).expect("small offsets")) } -} -/// For interrupt handlers: masks `irq` until it is serviced, then ends the -/// interrupt at the PICs (non-specific EOI, to both PICs for a slave IRQ). -pub fn mask_and_acknowledge(irq: u8) { - let (mut port, bit) = mask_register(irq); - // Safety: as in `unmask`; an EOI only clears the PIC's in-service bit. - unsafe { - let mask = port.read(); - port.write(mask | bit); - if irq >= 8 { - Port::::new(SLAVE_CMD).write(EOI); + fn read(&self, register: usize) -> u32 { + match self.xapic { + Some(registers) => registers.read_u32(register), + // Safety: reading a local APIC register has no side effects; the + // registers used are 32 bits. + #[expect(clippy::cast_possible_truncation, reason = "32-bit registers")] + None => unsafe { Self::msr(register).read() as u32 }, + } + } + + fn write(&self, register: usize, value: u32) { + match self.xapic { + Some(registers) => registers.write_u32(register, value), + // Safety: callers program the local APIC, which this kernel owns. + None => unsafe { Self::msr(register).write(u64::from(value)) }, + } + } + + /// For interrupt handlers: ends the interrupt. + pub fn end_of_interrupt(&self) { + self.write(reg::EOI, 0); + } + + /// The MSI message that raises `vector` on this CPU (fixed delivery, + /// edge). + /// + /// # Panics + /// + /// If the APIC ID does not fit an MSI address (above 255). + pub fn msi_message(&self, vector: u8) -> MsiMessage { + let id = match self.xapic { + Some(_) => self.read(reg::ID) >> 24, + None => self.read(reg::ID), + }; + // Without interrupt remapping, MSI addresses an 8-bit APIC ID. + assert!(id <= 0xff, "APIC ID {id} beyond MSI's reach"); + MsiMessage { + address: 0xfee0_0000 | (u64::from(id) << 12), + data: u32::from(vector), + } + } + + /// Raises the timer's vector once, when the TSC reaches `deadline` (at + /// once if it has; without TSC-deadline mode, a distant deadline may come + /// early); replaces a pending deadline. + pub fn arm_timer(&self, deadline: u64) { + match self.one_shot { + // Safety: programs this CPU's timer; zero would disarm. + None => unsafe { Msr::new(IA32_TSC_DEADLINE).write(deadline.max(1)) }, + Some((ticks, cycles)) => { + let delta = u128::from(deadline.saturating_sub(rdtsc())); + let count = delta * u128::from(ticks) / u128::from(cycles); + self.write( + reg::TIMER_INITIAL_COUNT, + u32::try_from(count.max(1)).unwrap_or(u32::MAX), + ); + } + } + } + + pub fn disarm_timer(&self) { + match self.one_shot { + // Safety: programs this CPU's timer. + None => unsafe { Msr::new(IA32_TSC_DEADLINE).write(0) }, + Some(_) => self.write(reg::TIMER_INITIAL_COUNT, 0), } - Port::::new(MASTER_CMD).write(EOI); } } diff --git a/litebox_hal/src/lib.rs b/litebox_hal/src/lib.rs index 9b3ac301df..df93381525 100644 --- a/litebox_hal/src/lib.rs +++ b/litebox_hal/src/lib.rs @@ -26,6 +26,4 @@ pub mod prk; #[cfg(target_arch = "x86_64")] pub mod rng; #[cfg(target_arch = "x86_64")] -pub mod timer; -#[cfg(target_arch = "x86_64")] pub mod virtio; diff --git a/litebox_hal/src/pci.rs b/litebox_hal/src/pci.rs index b5d262ae3d..9d400ee1a3 100644 --- a/litebox_hal/src/pci.rs +++ b/litebox_hal/src/pci.rs @@ -2,9 +2,10 @@ // Licensed under the MIT license. //! PCI: configuration space through the legacy mechanism #1 ports -//! (`0xCF8`/`0xCFC`), on bus 0. Firmware (SeaBIOS under QEMU) has already -//! assigned BARs and routed INTx to a legacy PIC IRQ. +//! (`0xCF8`/`0xCFC`), on bus 0, and MSI-X. Firmware (SeaBIOS under QEMU) has +//! already assigned BARs. +use crate::dma::Hal; use x86_64::instructions::port::Port; const CONFIG_ADDRESS: u16 = 0xcf8; @@ -21,8 +22,11 @@ const STATUS_CAPABILITIES: u16 = 1 << 4; /// Capabilities live past the standard header. const FIRST_CAPABILITY: u8 = 0x40; const BAR0: u8 = 0x10; -const INTERRUPT_LINE: u8 = 0x3c; -const INTERRUPT_PIN: u8 = 0x3d; + +const CAP_MSIX: u8 = 0x11; +const MSIX_CONTROL_ENABLE: u16 = 1 << 15; +const MSIX_CONTROL_FUNCTION_MASK: u16 = 1 << 14; +const MSIX_ENTRY_SIZE: usize = 16; /// `COMMAND` bits. pub const COMMAND_IO_SPACE: u16 = 1 << 0; @@ -100,16 +104,25 @@ impl Function { self.read_u16(COMMAND) } - /// A 16-bit write, leaving `STATUS` (write-1-to-clear) alone. + /// A 16-bit write, leaving the neighboring register (e.g., `STATUS`, + /// write-1-to-clear) alone. /// + /// # Safety + /// + /// As for [`Self::write_u32`]. + pub unsafe fn write_u16(self, offset: u8, value: u16) { + self.select(offset); + // Safety: forwarded to the caller. + unsafe { Port::::new(CONFIG_DATA + u16::from(offset & 2)).write(value) }; + } + /// # Safety /// /// Enabling bus mastering lets the device DMA wherever the driver points /// it. pub unsafe fn set_command(self, command: u16) { - self.select(COMMAND); // Safety: forwarded to the caller. - unsafe { Port::::new(CONFIG_DATA + u16::from(COMMAND & 2)).write(command) }; + unsafe { self.write_u16(COMMAND, command) }; } /// Memory BAR `index` (with its upper half, if 64-bit), sized; `None` if @@ -173,15 +186,68 @@ impl Function { } } - /// The legacy PIC IRQ that firmware routed INTx to, if any. - pub fn legacy_irq(self) -> Option { - if self.read_u8(INTERRUPT_PIN) == 0 { - return None; + /// Routes every MSI-X vector of the function to `message` and enables + /// MSI-X (which disables INTx); `false` if it has no usable MSI-X. + /// + /// # Safety + /// + /// Nothing may access the function's BARs meanwhile, and `message` must + /// raise an interrupt the kernel handles. + pub unsafe fn enable_msix(self, hal: &dyn Hal, message: MsiMessage) -> bool { + let mut msix = None; + self.for_each_capability(|offset, id| { + if id == CAP_MSIX && offset <= 0xf4 { + msix.get_or_insert(offset); + } + }); + let Some(cap) = msix else { + return false; + }; + let control = self.read_u16(cap + 2); + let entries = usize::from(control & 0x7ff) + 1; + let table = self.read_u32(cap + 4); + // Safety: the caller does not access the BARs meanwhile. + let Some(bar) = (unsafe { self.memory_bar(table.to_le_bytes()[0] & 7) }) else { + return false; + }; + let (offset, len) = (u64::from(table & !7), (entries * MSIX_ENTRY_SIZE) as u64); + if offset + len > bar.size { + return false; + } + // Safety: the table is within the BAR, so device registers. + let Some(table) = + (unsafe { hal.map_mmio(bar.address + offset, entries * MSIX_ENTRY_SIZE) }) + else { + return false; + }; + // Safety: decodes the BAR firmware assigned, so the table writes + // below reach the device. + unsafe { self.set_command(self.command() | COMMAND_MEMORY_SPACE) }; + #[expect(clippy::cast_possible_truncation, reason = "split into words")] + for entry in (0..entries).map(|entry| entry * MSIX_ENTRY_SIZE) { + table.write_u32(entry, message.address as u32); + table.write_u32(entry + 4, (message.address >> 32) as u32); + table.write_u32(entry + 8, message.data); + table.write_u32(entry + 12, 0); // unmasked } - Some(self.read_u8(INTERRUPT_LINE)).filter(|&irq| irq < 16) + // Safety: the vectors raise the caller's interrupt. + unsafe { + self.write_u16( + cap + 2, + (control | MSIX_CONTROL_ENABLE) & !MSIX_CONTROL_FUNCTION_MASK, + ); + } + true } } +/// An MSI message: a write of `data` to `address` raises the interrupt. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MsiMessage { + pub address: u64, + pub data: u32, +} + /// A memory BAR's physical range. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct MemoryBar { diff --git a/litebox_hal/src/timer.rs b/litebox_hal/src/timer.rs deleted file mode 100644 index f1547492f1..0000000000 --- a/litebox_hal/src/timer.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -//! Timer: PIT channel 0 in one-shot mode (mode 0), on PIC IRQ 0. Enough to -//! bound a halt; the TSC tells the time (see [`crate::clock`]). - -use x86_64::instructions::port::Port; - -/// The PIC IRQ the timer raises. -pub const IRQ: u8 = 0; - -const PIT_HZ: u64 = 1_193_182; -const PIT_CMD: u16 = 0x43; -const PIT_CH0: u16 = 0x40; -/// Channel 0, lobyte/hibyte, mode 0 (interrupt on terminal count), binary. -const CH0_ONESHOT: u8 = 0x30; - -/// Stops channel 0 without an interrupt; firmware may have left it -/// periodic. Call before unmasking [`IRQ`]. -pub fn disarm() { - // Safety: in mode 0, a control word holds OUT low and stops counting - // until a count is written. Nothing else uses channel 0. - unsafe { Port::::new(PIT_CMD).write(CH0_ONESHOT) }; -} - -/// Raises [`IRQ`] once, after `nanos`, clamped to about 55 ms (longer waits -/// re-arm). Replaces a pending shot. -pub fn arm_oneshot(nanos: u64) { - let ticks = u128::from(nanos) * u128::from(PIT_HZ) / 1_000_000_000; - let ticks = u16::try_from(ticks.clamp(1, 0xffff)).unwrap_or(u16::MAX); - let [lo, hi] = ticks.to_le_bytes(); - // Safety: programs channel 0 only; see `disarm`. - unsafe { - Port::::new(PIT_CMD).write(CH0_ONESHOT); - let mut ch0 = Port::::new(PIT_CH0); - ch0.write(lo); - ch0.write(hi); - } -} diff --git a/litebox_hal/src/virtio/console.rs b/litebox_hal/src/virtio/console.rs index 183c27bec1..555c1e4dbf 100644 --- a/litebox_hal/src/virtio/console.rs +++ b/litebox_hal/src/virtio/console.rs @@ -22,6 +22,7 @@ use super::Error; use super::pci::PciTransport; use super::queue::{Buffer, VirtQueue}; use crate::dma::{DmaRegion, Hal}; +use crate::pci::MsiMessage; use alloc::collections::VecDeque; use alloc::vec; use alloc::vec::Vec; @@ -108,15 +109,22 @@ pub struct VirtioConsole { } impl VirtioConsole { - /// The first virtio console on bus 0, brought up; `None` if there is - /// none. + /// The first virtio console on bus 0, brought up, interrupting with + /// `interrupt` (if it can; see [`PciTransport::interrupts`]); `None` if + /// there is none. /// /// # Errors /// /// The device cannot be driven. - pub fn probe(hal: &'static dyn Hal) -> Option> { + pub fn probe( + hal: &'static dyn Hal, + interrupt: Option, + ) -> Option> { let function = super::find_pci_device(DEVICE_TYPE)?; - Some(PciTransport::new(hal, function).and_then(|transport| Self::bring_up(hal, transport))) + Some( + PciTransport::new(hal, function, interrupt) + .and_then(|transport| Self::bring_up(hal, transport)), + ) } fn bring_up(hal: &'static dyn Hal, transport: PciTransport) -> Result { diff --git a/litebox_hal/src/virtio/mod.rs b/litebox_hal/src/virtio/mod.rs index 9875a84367..bf0ac909d4 100644 --- a/litebox_hal/src/virtio/mod.rs +++ b/litebox_hal/src/virtio/mod.rs @@ -3,8 +3,8 @@ //! Virtio 1.x devices over PCI ([`pci::PciTransport`]): the modern //! interface, with its structures in memory BARs (mapped through -//! [`crate::dma::Hal`]) and INTx interrupts (routed to a legacy PIC IRQ by -//! firmware; MSI-X is left disabled). Split virtqueues ([`queue`]). +//! [`crate::dma::Hal`]) and interrupts by MSI-X. Split virtqueues +//! ([`queue`]). //! //! Devices: [`console`]. The transport and queues are device-independent, //! for further devices (e.g., file systems and networking for the broker). @@ -34,7 +34,7 @@ pub fn find_pci_device(device_type: u16) -> Option { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { - /// The device lacks a required structure (common, notify, or ISR). + /// The device lacks a required structure (common or notify). MissingStructure(&'static str), /// A structure is outside its BAR, too short, or not in a memory BAR. BadStructure(&'static str), diff --git a/litebox_hal/src/virtio/pci.rs b/litebox_hal/src/virtio/pci.rs index bd286ef2e3..bf190f5300 100644 --- a/litebox_hal/src/virtio/pci.rs +++ b/litebox_hal/src/virtio/pci.rs @@ -2,12 +2,12 @@ // Licensed under the MIT license. //! The modern virtio-PCI transport: structures located by vendor-specific -//! capabilities, in memory BARs. +//! capabilities, in memory BARs; interrupts by MSI-X. use super::Error; use super::queue::VirtQueue; use crate::dma::{Hal, Mmio}; -use crate::pci::{self, Function, MemoryBar}; +use crate::pci::{self, Function, MemoryBar, MsiMessage}; const CAP_VENDOR_SPECIFIC: u8 = 0x09; @@ -27,7 +27,6 @@ mod cap { pub const COMMON: u8 = 1; pub const NOTIFY: u8 = 2; - pub const ISR: u8 = 3; pub const DEVICE: u8 = 4; } @@ -40,6 +39,7 @@ mod common { pub const DEVICE_STATUS: usize = 0x14; pub const QUEUE_SELECT: usize = 0x16; pub const QUEUE_SIZE: usize = 0x18; + pub const QUEUE_MSIX_VECTOR: usize = 0x1a; pub const QUEUE_ENABLE: usize = 0x1c; pub const QUEUE_NOTIFY_OFF: usize = 0x1e; pub const QUEUE_DESC: usize = 0x20; @@ -72,7 +72,6 @@ struct Location { struct Locations { common: Option, notify: Option<(Location, u32)>, - isr: Option, device: Option, } @@ -97,9 +96,6 @@ impl Locations { let multiplier = function.read_u32(offset + cap::NOTIFY_OFF_MULTIPLIER); found.notify.get_or_insert((location, multiplier)); } - cap::ISR => { - found.isr.get_or_insert(location); - } cap::DEVICE => { found.device.get_or_insert(location); } @@ -116,19 +112,23 @@ pub struct PciTransport { common: Mmio, notify: Mmio, notify_off_multiplier: u32, - isr: Mmio, device: Option, - irq: Option, + /// Whether the device interrupts (by MSI-X); else it is polled. + interrupts: bool, } impl PciTransport { - /// Maps `function`'s structures and enables its memory decoding, bus - /// mastering, and INTx. + /// Maps `function`'s structures, enables its memory decoding and bus + /// mastering, and sends its interrupts as `interrupt` if it has MSI-X. /// /// # Errors /// /// A missing or malformed structure, or one that cannot be mapped. - pub fn new(hal: &dyn Hal, function: Function) -> Result { + pub fn new( + hal: &dyn Hal, + function: Function, + interrupt: Option, + ) -> Result { let locations = Locations::read(function); let mut bars = [None::; 6]; let mut map = |location: Location, what, min_len: usize| -> Result { @@ -150,34 +150,38 @@ impl PciTransport { let common = locations.common.ok_or(Error::MissingStructure("common"))?; let (notify, notify_off_multiplier) = locations.notify.ok_or(Error::MissingStructure("notify"))?; - let isr = locations.isr.ok_or(Error::MissingStructure("ISR"))?; - let transport = Self { - function, - common: map(common, "common", common::LEN)?, - notify: map(notify, "notify", 2)?, - notify_off_multiplier, - isr: map(isr, "ISR", 1)?, - device: locations - .device - .map(|device| map(device, "device", 0)) - .transpose()?, - irq: function.legacy_irq(), - }; + let common = map(common, "common", common::LEN)?; + let notify = map(notify, "notify", 2)?; + let device = locations + .device + .map(|device| map(device, "device", 0)) + .transpose()?; + // Safety: the driver does not access the BARs yet; the message is the + // caller's interrupt. + let interrupts = + interrupt.is_some_and(|message| unsafe { function.enable_msix(hal, message) }); let command = (function.command() | pci::COMMAND_MEMORY_SPACE | pci::COMMAND_BUS_MASTER) - & !pci::COMMAND_INTX_DISABLE; + | pci::COMMAND_INTX_DISABLE; // Safety: decodes the BARs firmware assigned; DMA only goes where the // driver points the queues. unsafe { function.set_command(command) }; - Ok(transport) + Ok(Self { + function, + common, + notify, + notify_off_multiplier, + device, + interrupts, + }) } pub fn function(&self) -> Function { self.function } - /// The PIC IRQ of the device's INTx, if firmware routed one. - pub fn irq(&self) -> Option { - self.irq + /// Whether the device interrupts (by MSI-X); else it must be polled. + pub fn interrupts(&self) -> bool { + self.interrupts } /// The device-specific configuration, if any. @@ -240,12 +244,13 @@ impl PciTransport { Ok(accepted) } - /// Sets up and enables queue `index`, with at most `max_size` entries. + /// Sets up and enables queue `index`, with at most `max_size` entries, + /// interrupting on MSI-X vector 0 if the device interrupts. /// /// # Errors /// - /// The device lacks the queue, its notification address is out of - /// range, or memory ran out. + /// The device lacks the queue, its notification address or vector is + /// unusable, or memory ran out. pub fn setup_queue( &self, hal: &'static dyn Hal, @@ -264,6 +269,13 @@ impl PciTransport { .checked_mul(usize::try_from(self.notify_off_multiplier).unwrap_or(usize::MAX)) .filter(|&offset| offset.is_multiple_of(2) && offset + 2 <= self.notify.len()) .ok_or(Error::BadNotifyOffset(index))?; + if self.interrupts { + self.common.write_u16(common::QUEUE_MSIX_VECTOR, 0); + // The device answers `NO_VECTOR` (0xffff) if it cannot. + if self.common.read_u16(common::QUEUE_MSIX_VECTOR) != 0 { + return Err(Error::QueueUnavailable(index)); + } + } let queue = VirtQueue::new(hal, index, size, notify_offset)?; let [desc, driver, device] = queue.addresses(); self.common.write_u16(common::QUEUE_SIZE, size); @@ -292,11 +304,4 @@ impl PciTransport { core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); self.notify.write_u16(queue.notify_offset(), queue.index()); } - - /// Acknowledges the device's interrupt (deasserting INTx); returns the - /// ISR status (bit 0: a queue, bit 1: configuration). Process the queues - /// afterwards, so nothing that arrived before goes unnoticed. - pub fn acknowledge_interrupt(&self) -> u8 { - self.isr.read_u8(0) - } } diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs index 5536200824..257a22725e 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs @@ -5,7 +5,7 @@ //! NMI entry cannot assume kernel GS or a completed syscall stack switch. //! CR4.MCE remains off; machine checks shut down the CPU. -use core::sync::atomic::{AtomicU64, Ordering}; +use crate::InterruptHandler; use litebox_common_linux::PtRegs; use spin::Once; use x86_64::{VirtAddr, structures::idt::InterruptDescriptorTable}; @@ -42,10 +42,7 @@ const NMI_IST_INDEX: u16 = 1; struct Interrupts { idt: InterruptDescriptorTable, /// See [`crate::set_interrupt_handler`]. - handler: Once bool>, - /// Bit `v % 64` of word `v / 64`: vector `v` was handled since it was - /// last taken ([`take_pending`]). - pending: [AtomicU64; 4], + handler: Once<&'static InterruptHandler>, } static INTERRUPTS: Once = Once::new(); @@ -55,7 +52,6 @@ fn idt(ignored_vectors: &[u8]) -> &'static InterruptDescriptorTable { .call_once(|| Interrupts { idt: build_idt(ignored_vectors), handler: Once::new(), - pending: [const { AtomicU64::new(0) }; 4], }) .idt } @@ -128,7 +124,7 @@ pub fn init_idt(ignored_vectors: &[u8]) { /// # Panics /// /// Before [`init_idt`], or if a handler is already set. -pub(crate) fn set_external_interrupt_handler(handler: fn(u8) -> bool) { +pub(crate) fn set_external_interrupt_handler(handler: &'static InterruptHandler) { let interrupts = INTERRUPTS.get().expect("the IDT is initialized at boot"); let mut installed = false; interrupts.handler.call_once(|| { @@ -138,13 +134,6 @@ pub(crate) fn set_external_interrupt_handler(handler: fn(u8) -> bool) { assert!(installed, "an interrupt handler is already set"); } -/// The vectors handled since the last call, as a 256-bit set. -pub(crate) fn take_pending() -> [u64; 4] { - INTERRUPTS.get().map_or([0; 4], |interrupts| { - core::array::from_fn(|word| interrupts.pending[word].swap(0, Ordering::Relaxed)) - }) -} - /// Runs with interrupts disabled, on whatever stack and GS the interrupted /// context had (see `isr_external`). #[unsafe(no_mangle)] @@ -155,8 +144,6 @@ extern "C" fn external_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) { .get() .is_some_and(|handler| handler(vector)) { - interrupts.pending[usize::from(vector / 64)] - .fetch_or(1 << (vector % 64), Ordering::Relaxed); return; } panic!("EXCEPTION: UNEXPECTED INTERRUPT (vector {vector:#x}, RIP {rip:#x}, CS {cs:#x})"); diff --git a/litebox_platform_vm_kernel/src/lib.rs b/litebox_platform_vm_kernel/src/lib.rs index 63fd2c0f84..d6f581535d 100644 --- a/litebox_platform_vm_kernel/src/lib.rs +++ b/litebox_platform_vm_kernel/src/lib.rs @@ -97,6 +97,9 @@ pub struct BootConfig<'a> { pub ignored_vectors: &'a [u8], } +/// See [`set_interrupt_handler`]. +pub type InterruptHandler = dyn Fn(u8) -> bool + Sync; + /// Installs the handler for external interrupts (vectors 32 and up, except /// [`BootConfig::ignored_vectors`]), which returns whether it handled the /// vector; an unhandled one panics. Before this, every external interrupt @@ -104,30 +107,16 @@ pub struct BootConfig<'a> { /// /// The handler runs with interrupts disabled, on the interrupted context's GS /// (user mode, or [`VmKernel::halt_until_interrupt`]): it must not touch -/// per-CPU data, block, or take locks, only quiet the source. The kernel acts -/// on handled vectors later ([`VmKernel::take_pending_interrupts`]). +/// per-CPU data, block, or take locks; it should only end the interrupt and +/// leave the work to the kernel after the halt. /// /// # Panics /// /// On a second call. -pub fn set_interrupt_handler(handler: fn(u8) -> bool) { +pub fn set_interrupt_handler(handler: &'static InterruptHandler) { arch::interrupts::set_external_interrupt_handler(handler); } -/// A set of interrupt vectors. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct InterruptVectors([u64; 4]); - -impl InterruptVectors { - pub fn contains(&self, vector: u8) -> bool { - self.0[usize::from(vector / 64)] & (1 << (vector % 64)) != 0 - } - - pub fn iter(&self) -> impl Iterator + '_ { - (0..=u8::MAX).filter(|&vector| self.contains(vector)) - } -} - /// Valid until unregistered; its representation is private to the platform. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct AddressSpaceId(usize); @@ -475,11 +464,6 @@ impl VmKernel { })) } - /// The vectors the interrupt handler handled since the last call. - pub fn take_pending_interrupts(&self) -> InterruptVectors { - InterruptVectors(arch::interrupts::take_pending()) - } - /// Halts until an interrupt is handled, with interrupts enabled only for /// the halt. An interrupt that became pending while they were disabled /// ends the halt at once, so check for work first and halt after: none diff --git a/litebox_runner_vm_kernel/src/devices.rs b/litebox_runner_vm_kernel/src/devices.rs index b8217af896..8af3e9a902 100644 --- a/litebox_runner_vm_kernel/src/devices.rs +++ b/litebox_runner_vm_kernel/src/devices.rs @@ -6,11 +6,11 @@ //! //! - Runner processes' standard streams: a virtio console if there is one, //! else output to the serial console and no input. -//! - Events: the console's interrupt, and the PIT for deadlines. +//! - Events: the console's interrupt (MSI-X), and the local APIC's timer +//! for deadlines. //! -//! The interrupt handler only masks the IRQ (the platform records the -//! vector); the kernel services the device and unmasks it when it next polls -//! or halts. +//! The interrupt handler only ends the interrupt; the kernel services the +//! device when it next polls or halts. use alloc::boxed::Box; use core::sync::atomic::{AtomicBool, Ordering}; @@ -19,8 +19,8 @@ use litebox_broker_core::stdio::{ StdioOutputStream, StdioProvider, StdioProviderError, StdioStream, }; use litebox_broker_protocol::readiness::ReadinessFlags; +use litebox_hal::interrupt::LocalApic; use litebox_hal::virtio::console::VirtioConsole; -use litebox_hal::{interrupt, timer}; use litebox_platform_vm_kernel::VmKernel; use spin::Mutex; @@ -28,19 +28,13 @@ use spin::Mutex; /// the rest is dropped. const WRITE_TIMEOUT_NANOS: u64 = 100_000_000; -/// See `litebox_platform_vm_kernel::set_interrupt_handler`: no locks, no -/// per-CPU data. The IRQ stays masked until [`Devices::acknowledge`]. -fn on_interrupt(vector: u8) -> bool { - let Some(irq) = interrupt::irq(vector) else { - return false; - }; - interrupt::mask_and_acknowledge(irq); - true -} +const TIMER_VECTOR: u8 = 0x30; +const CONSOLE_VECTOR: u8 = 0x31; struct Console { device: Mutex, - irq: Option, + /// Whether it interrupts; else it is polled. + interrupts: bool, /// Standard-input files, to wake when input arrives. stdin_watchers: Mutex, /// Whether input was pending when last looked at; watchers are woken @@ -56,6 +50,7 @@ impl Console { pub struct Devices { platform: &'static VmKernel, + apic: LocalApic, tsc_khz: u64, console: Option, } @@ -63,10 +58,9 @@ pub struct Devices { impl Devices { /// Brings up the devices and enables their interrupts. Call once. pub fn init(platform: &'static VmKernel, tsc_khz: u64) -> &'static Self { - litebox_platform_vm_kernel::set_interrupt_handler(on_interrupt); - timer::disarm(); - interrupt::unmask(timer::IRQ); - let console = match VirtioConsole::probe(platform) { + let apic = LocalApic::init(platform, TIMER_VECTOR, tsc_khz); + let interrupt = apic.msi_message(CONSOLE_VECTOR); + let console = match VirtioConsole::probe(platform, Some(interrupt)) { None => { litebox_util_log::info!( "no virtio console: standard output goes to the serial console" @@ -81,60 +75,51 @@ impl Devices { None } Some(Ok(device)) => { - let irq = device.transport().irq(); + let interrupts = device.transport().interrupts(); litebox_util_log::info!( pci:% = device.transport().function(), - irq:? = irq; + interrupts:% = interrupts; "virtio console: standard streams" ); - if let Some(irq) = irq { - interrupt::unmask(irq); - } Some(Console { device: Mutex::new(device), - irq, + interrupts, stdin_watchers: Mutex::new(ReadinessWatchers::default()), stdin_readable: AtomicBool::new(false), }) } }; - Box::leak(Box::new(Self { + let devices: &'static Self = Box::leak(Box::new(Self { platform, + apic, tsc_khz, console, - })) + })); + litebox_platform_vm_kernel::set_interrupt_handler(Box::leak(Box::new(|vector| { + devices.on_interrupt(vector) + }))); + devices } - /// Acknowledges recorded interrupts at their devices and unmasks them. - fn acknowledge(&self) { - let pending = self.platform.take_pending_interrupts(); - if let Some(console) = &self.console - && let Some(irq) = console.irq - && pending.contains(interrupt::vector(irq)) - { - // Before unmasking, or the line would still be asserted. - console.device.lock().transport().acknowledge_interrupt(); - } - for irq in pending.iter().filter_map(interrupt::irq) { - interrupt::unmask(irq); + /// See `litebox_platform_vm_kernel::set_interrupt_handler`: no locks, no + /// per-CPU data. + fn on_interrupt(&self, vector: u8) -> bool { + if !matches!(vector, TIMER_VECTOR | CONSOLE_VECTOR) { + return false; } + self.apic.end_of_interrupt(); + true } /// Halts until an interrupt, or until `deadline` (TSC). fn halt(&self, deadline: Option) { if let Some(deadline) = deadline { - let now = rdtsc(); - if now >= deadline { - return; - } - let nanos = u128::from(deadline - now) * 1_000_000 / u128::from(self.tsc_khz); - timer::arm_oneshot(u64::try_from(nanos).unwrap_or(u64::MAX)); + self.apic.arm_timer(deadline); } self.platform.halt_until_interrupt(); if deadline.is_some() { - timer::disarm(); + self.apic.disarm_timer(); } - self.acknowledge(); } } @@ -145,7 +130,6 @@ fn rdtsc() -> u64 { impl litebox_broker_vm_kernel::Events for Devices { fn poll(&self) { - self.acknowledge(); // Wake watchers when input starts to be pending. if let Some(console) = &self.console { let readable = console.readable(); @@ -161,7 +145,7 @@ impl litebox_broker_vm_kernel::Events for Devices { let interrupts = self .console .as_ref() - .is_some_and(|console| console.irq.is_some()); + .is_some_and(|console| console.interrupts); if deadline.is_none() && !interrupts { return false; } From eb2f57f8988418432d92c4135afedcae5a89af94 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Sat, 10 Oct 2026 02:22:45 +0000 Subject: [PATCH 3/7] fix notification and script --- Cargo.lock | 2 - dev_tools/qemu_test_lib.sh | 10 +- dev_tools/run_optee_on_qemu.sh | 1 + dev_tools/run_optee_on_vm_userland.sh | 1 + litebox_bootloader/src/lib.rs | 4 +- litebox_broker_vm_kernel/src/lib.rs | 67 ++++++---- litebox_broker_vm_kernel/src/readiness.rs | 12 +- litebox_broker_vm_kernel/src/stdio.rs | 99 +++++++++++++++ litebox_hal/src/clock.rs | 61 ++++++++- litebox_hal/src/interrupt.rs | 8 +- litebox_hal/src/pci.rs | 8 +- litebox_hal/src/virtio/console.rs | 2 +- litebox_hal/src/virtio/pci.rs | 3 +- litebox_runner_vm_kernel/Cargo.toml | 2 - litebox_runner_vm_kernel/src/devices.rs | 144 ++++++---------------- litebox_runner_vm_kernel/src/main.rs | 2 +- 16 files changed, 267 insertions(+), 159 deletions(-) create mode 100644 litebox_broker_vm_kernel/src/stdio.rs diff --git a/Cargo.lock b/Cargo.lock index e2f16a00d0..fa3c8369ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2067,8 +2067,6 @@ dependencies = [ "arrayvec", "base64", "litebox_bootloader", - "litebox_broker_core", - "litebox_broker_protocol", "litebox_broker_vm_kernel", "litebox_common_optee", "litebox_common_vm_abi", diff --git a/dev_tools/qemu_test_lib.sh b/dev_tools/qemu_test_lib.sh index b63a4cda30..08db8332e9 100644 --- a/dev_tools/qemu_test_lib.sh +++ b/dev_tools/qemu_test_lib.sh @@ -4,16 +4,18 @@ # Licensed under the MIT license. # Shared by the dev_tools/run_optee_on_*.sh scripts; source it. Reads QEMU, -# QEMU_ACCEL, TIMEOUT, LITEBOX_LOG and CARGO_BUILD (see the scripts), makes a -# relative CARGO_TARGET_DIR absolute, and creates $WORK, removed on exit. +# QEMU_ACCEL, QEMU_CPU, TIMEOUT, LITEBOX_LOG and CARGO_BUILD (see the scripts), +# makes a relative CARGO_TARGET_DIR absolute, and creates $WORK, removed on +# exit. QEMU=${QEMU:-qemu-system-x86_64} if [[ -z ${QEMU_ACCEL:-} ]]; then if [[ -r /dev/kvm && -w /dev/kvm ]]; then QEMU_ACCEL=kvm; else QEMU_ACCEL=tcg; fi fi case $QEMU_ACCEL in - kvm) CPU=host ;; - tcg) CPU=max ;; + # invtsc (not migratable) lets QEMU publish the TSC frequency to the guest. + kvm) CPU=${QEMU_CPU:-host,+invtsc} ;; + tcg) CPU=${QEMU_CPU:-max} ;; *) echo "error: unknown QEMU_ACCEL '$QEMU_ACCEL'" >&2; exit 2 ;; esac TIMEOUT=${TIMEOUT:-120} diff --git a/dev_tools/run_optee_on_qemu.sh b/dev_tools/run_optee_on_qemu.sh index 5e66f6a360..40fb8aa031 100755 --- a/dev_tools/run_optee_on_qemu.sh +++ b/dev_tools/run_optee_on_qemu.sh @@ -17,6 +17,7 @@ # Environment: # QEMU QEMU binary/command (default: qemu-system-x86_64) # QEMU_ACCEL kvm or tcg (default: kvm if /dev/kvm is usable, else tcg) +# QEMU_CPU QEMU CPU model (default: host,+invtsc for kvm, max for tcg) # TIMEOUT per-run timeout in seconds (default: 120) # LITEBOX_LOG guest log level, e.g. debug (default: info) # CARGO_BUILD command that builds the runner, run in its directory diff --git a/dev_tools/run_optee_on_vm_userland.sh b/dev_tools/run_optee_on_vm_userland.sh index 2dd7ce13f9..4219de4c56 100755 --- a/dev_tools/run_optee_on_vm_userland.sh +++ b/dev_tools/run_optee_on_vm_userland.sh @@ -29,6 +29,7 @@ # Environment: # QEMU QEMU binary/command (default: qemu-system-x86_64) # QEMU_ACCEL kvm or tcg (default: kvm if /dev/kvm is usable, else tcg) +# QEMU_CPU QEMU CPU model (default: host,+invtsc for kvm, max for tcg) # TIMEOUT per-run timeout in seconds (default: 120) # LITEBOX_LOG guest log level, e.g. debug (default: info) # CARGO_BUILD command that builds the runners (and the rewriter unless diff --git a/litebox_bootloader/src/lib.rs b/litebox_bootloader/src/lib.rs index 27faf0350e..b3450504d5 100644 --- a/litebox_bootloader/src/lib.rs +++ b/litebox_bootloader/src/lib.rs @@ -101,8 +101,8 @@ fn kernel_start(boot_info: impl FnOnce() -> BootInfo) -> ! { .collect(); let to_pa = |va: u64| VmKernel::va_to_pa(x86_64::VirtAddr::new(va)); litebox_hal::interrupt::init_legacy_pics(); - let tsc_khz = litebox_hal::clock::calibrate_tsc_khz(); - litebox_util_log::info!(mhz:% = tsc_khz / 1000; "TSC calibrated against the PIT"); + let (tsc_khz, source) = litebox_hal::clock::tsc_khz(); + litebox_util_log::info!(mhz:% = tsc_khz / 1000, source:? = source; "TSC frequency"); BOOT.call_once(|| (info, tsc_khz)); let clock = alloc::boxed::Box::leak(alloc::boxed::Box::new(TscClock::new(tsc_khz))); // Safety: the front end established the direct mapping with IRQs off on one diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index dcf38eedbe..f4f1de4f60 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -8,10 +8,10 @@ //! kernel's mapping; payloads are in the process's lazily populated shared //! buffers. //! -//! Files are the devices under `/dev`, with standard streams from the -//! runner's [`StdioProvider`]. Readiness (e.g., of standard input) goes to -//! the association's notification ring. Waiting ([`BrokerEnterOp::Wait`]) -//! halts until the runner's [`Events`] report something, e.g., an interrupt. +//! Files are the devices under `/dev`, with standard streams on the +//! runner's [`stdio::Console`]. Readiness (e.g., of standard input) goes to the +//! association's notification ring. Waiting ([`BrokerEnterOp::Wait`]) halts +//! until the runner's [`Events`] report something, e.g., an interrupt. #![cfg(target_arch = "x86_64")] #![no_std] @@ -22,6 +22,7 @@ extern crate alloc; mod memory; mod providers; mod readiness; +pub mod stdio; use alloc::boxed::Box; use alloc::sync::Arc; @@ -30,7 +31,6 @@ use core::cell::RefCell; use core::convert::Infallible; use litebox_broker_core::BrokerCore; use litebox_broker_core::readiness::ReadinessSink; -use litebox_broker_core::stdio::StdioProvider; use litebox_broker_host::BrokerHostAssociation; use litebox_broker_protocol::message::{BrokerHandshakeRequest, BrokerHandshakeResponse}; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SHARED_BUFFER_POOL_SIZE}; @@ -47,6 +47,7 @@ use litebox_platform_vm_kernel::PinnedUserPages; use memory::{PinnedControlRing, UserSharedMemory}; use readiness::KernelReadiness; +use stdio::ConsoleStdio; /// Size of a process's broker shared memory. pub const SHARED_MEMORY_SIZE: usize = SHARED_BUFFER_POOL_SIZE; @@ -57,20 +58,16 @@ pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; /// What the broker waits for: the runner's event sources (e.g., device /// interrupts and a timer). pub trait Events: Sync { - /// Acts on events since the last call, e.g., publishes the readiness of - /// input that arrived. - fn poll(&self); - /// Halts until an event may have happened, or until `deadline` (TSC) - /// passes; then [`Self::poll`] runs. Without a deadline, false if no - /// event could ever end the halt (and it does not halt). + /// passes. Without a deadline, false if no event could ever end the halt + /// (and it does not halt). fn wait(&self, deadline: Option) -> bool; } /// The runner's sources for a [`Broker`]. pub struct Config { /// Behind `/dev/stdin`, `/dev/stdout`, and `/dev/stderr`. - pub stdio: Arc, + pub console: &'static dyn stdio::Console, pub events: &'static dyn Events, } @@ -79,6 +76,7 @@ pub struct Config { pub struct Broker { core: BrokerCore, events: &'static dyn Events, + stdio: Arc, } impl Broker { @@ -88,9 +86,11 @@ impl Broker { /// /// Without a hardware CSPRNG, or on a second call. pub fn new(config: Config) -> Self { + let stdio = Arc::new(ConsoleStdio::new(config.console)); Self { - core: providers::core(config.stdio), + core: providers::core(stdio.clone()), events: config.events, + stdio, } } @@ -254,7 +254,8 @@ impl Association { /// changes or the deadline passes. Only notifications change ring words /// asynchronously; a wait on another word without a deadline, or on the /// notification epoch when no event could publish, is - /// [`Status::Stalled`]. + /// [`Status::Stalled`]. A corrupted notification ring fails the + /// association, as a corrupted request ring does. /// /// # Errors /// @@ -264,6 +265,20 @@ impl Association { let State::Active(active) = &mut *state else { return Err(EnterError::Status(Status::Denied)); }; + let result = self.enter_active(active, request); + if result == Err(EnterError::Failed) + && let State::Active(active) = core::mem::replace(&mut *state, State::Ended) + { + end(active.association, &self.readiness); + } + result + } + + fn enter_active( + &self, + active: &mut Active, + request: &BrokerEnterRequest, + ) -> Result<(), EnterError> { let invalid = || EnterError::Status(Status::InvalidArgument); let wait_offset = match request.op { BrokerEnterOp::Wake => { @@ -280,14 +295,9 @@ impl Association { Some(offset) } }; - self.broker.events.poll(); - if active.serve().is_err() { - if let State::Active(active) = core::mem::replace(&mut *state, State::Ended) { - end(active.association, &self.readiness); - } - return Err(EnterError::Failed); - } - self.readiness.flush(); + self.broker.stdio.poll(); + active.serve().map_err(|Failed| EnterError::Failed)?; + self.flush_readiness()?; let Some(offset) = wait_offset else { return Ok(()); }; @@ -310,8 +320,19 @@ impl Association { if !self.broker.events.wait(deadline) { return Err(EnterError::Status(Status::Stalled)); } - self.readiness.flush(); + self.broker.stdio.poll(); + self.flush_readiness()?; + } + } + + /// Publishes pending readiness; a corrupted notification ring fails the + /// association. + fn flush_readiness(&self) -> Result<(), EnterError> { + self.readiness.flush(); + if self.readiness.failed() { + return Err(EnterError::Failed); } + Ok(()) } } diff --git a/litebox_broker_vm_kernel/src/readiness.rs b/litebox_broker_vm_kernel/src/readiness.rs index e31788ff38..6646e7acca 100644 --- a/litebox_broker_vm_kernel/src/readiness.rs +++ b/litebox_broker_vm_kernel/src/readiness.rs @@ -5,6 +5,7 @@ //! at once, whichever process is current. What a full ring cannot take //! stays pending until the next [`KernelReadiness::flush`]. +use core::sync::atomic::{AtomicBool, Ordering}; use litebox_broker_core::readiness::ReadinessSink; use litebox_broker_core::{BrokerError, Result}; use litebox_broker_host::readiness::{ @@ -29,6 +30,8 @@ pub(crate) struct KernelReadiness { publisher: ReadinessPublisher, /// Until the handshake and after the association ends: `None`. notifications: SpinMutex>, + /// The process corrupted its notification ring. + failed: AtomicBool, } impl KernelReadiness { @@ -55,14 +58,19 @@ impl KernelReadiness { match publish_pending_readiness(&self.publisher, &mut Ring(producer)) { Ok(()) | Err(RingError::Full) => {} Err(RingError::Ring(error)) => { - // The process corrupted its ring; its next broker entry fails - // the association. log::warn!("broker notification ring failed: {error:?}"); *notifications = None; self.publisher.close(); + self.failed.store(true, Ordering::Relaxed); } } } + + /// Whether the process corrupted its notification ring, which fails the + /// association. + pub(crate) fn failed(&self) -> bool { + self.failed.load(Ordering::Relaxed) + } } enum RingError { diff --git a/litebox_broker_vm_kernel/src/stdio.rs b/litebox_broker_vm_kernel/src/stdio.rs new file mode 100644 index 0000000000..ee1969a327 --- /dev/null +++ b/litebox_broker_vm_kernel/src/stdio.rs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Runner processes' standard streams, on the runner's [`Console`]; both +//! output streams share it. Standard-input files are woken when input starts +//! to be pending. + +use core::sync::atomic::{AtomicBool, Ordering}; +use litebox_broker_core::readiness::{ReadinessRegistration, ReadinessWatchers}; +use litebox_broker_core::stdio::{ + StdioOutputStream, StdioProvider, StdioProviderError, StdioStream, +}; +use litebox_broker_protocol::readiness::ReadinessFlags; +use spin::Mutex; + +/// A byte-stream device behind the standard streams. +pub trait Console: Sync { + /// Copies received bytes into `output`: how many (zero if none yet), or + /// `None` at the end of input (e.g., a device without input). + fn read(&self, output: &mut [u8]) -> Option; + + /// Whether [`Self::read`] would not return zero: input or its end. + fn has_input(&self) -> bool; + + /// How much of `input` the device took; less if it stays full. + fn write(&self, input: &[u8]) -> usize; +} + +pub(crate) struct ConsoleStdio { + console: &'static dyn Console, + /// Standard-input files, to wake when input arrives. + watchers: Mutex, + /// Whether input was pending when last looked at. + readable: AtomicBool, +} + +impl ConsoleStdio { + pub(crate) fn new(console: &'static dyn Console) -> Self { + Self { + console, + watchers: Mutex::new(ReadinessWatchers::default()), + readable: AtomicBool::new(false), + } + } + + /// Wakes standard-input watchers if input started to be pending. + pub(crate) fn poll(&self) { + let readable = self.console.has_input(); + if readable && !self.readable.swap(true, Ordering::Relaxed) { + self.watchers.lock().publish(ReadinessFlags::READ); + } else if !readable { + self.readable.store(false, Ordering::Relaxed); + } + } +} + +/// Never a terminal. +impl StdioProvider for ConsoleStdio { + fn read(&self, output: &mut [u8]) -> Result { + match self.console.read(output) { + None => Ok(0), + Some(0) => Err(StdioProviderError::WouldBlock), + Some(read) => Ok(read), + } + } + + fn write(&self, _stream: StdioOutputStream, input: &[u8]) -> Result { + match self.console.write(input) { + 0 => Err(StdioProviderError::WouldBlock), + written => Ok(written), + } + } + + fn is_terminal(&self, _stream: StdioStream) -> bool { + false + } + + fn readiness(&self, stream: StdioStream) -> ReadinessFlags { + match stream { + StdioStream::Stdin if !self.console.has_input() => ReadinessFlags(0), + StdioStream::Stdin => ReadinessFlags::READ, + StdioStream::Stdout | StdioStream::Stderr => ReadinessFlags::WRITE, + } + } + + fn watch( + &self, + stream: StdioStream, + registration: &ReadinessRegistration, + ) -> litebox_broker_core::Result<()> { + if stream == StdioStream::Stdin { + self.watchers.lock().watch(registration)?; + if self.console.has_input() { + registration.publish(ReadinessFlags::READ)?; + } + } + Ok(()) + } +} diff --git a/litebox_hal/src/clock.rs b/litebox_hal/src/clock.rs index d80877bee0..014986cc73 100644 --- a/litebox_hal/src/clock.rs +++ b/litebox_hal/src/clock.rs @@ -1,18 +1,71 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Clocks: TSC frequency, calibrated against the legacy PIT. +//! Clocks: the TSC's frequency, from the most authoritative source the +//! machine has: CPUID, the hypervisor, or else calibration against the +//! legacy PIT (which, e.g., Firecracker and Cloud Hypervisor lack). + +use core::arch::x86_64::__cpuid_count as cpuid; const PIT_HZ: u64 = 1_193_182; const CALIBRATION_MS: u64 = 50; const CALIBRATION_TIMEOUT_CYCLES: u64 = 1 << 36; -/// Uses PIT channel 2 exclusively for ~50 ms. +/// Frequencies outside this range are not believed. +const PLAUSIBLE_KHZ: core::ops::RangeInclusive = 100_000..=100_000_000; + +/// Where [`tsc_khz`] got the frequency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TscSource { + /// CPUID leaf 0x15: the core crystal and its ratio (Intel). + Cpuid, + /// The hypervisor's timing leaf 0x40000010 (KVM via QEMU, VMware). + Hypervisor, + /// Calibration against the PIT. + Pit, +} + +/// The TSC frequency in kHz, and its source. /// /// # Panics /// -/// Panics if calibration fails. -pub fn calibrate_tsc_khz() -> u64 { +/// If no source works. +pub fn tsc_khz() -> (u64, TscSource) { + if let Some(khz) = cpuid_khz().filter(|khz| PLAUSIBLE_KHZ.contains(khz)) { + return (khz, TscSource::Cpuid); + } + if let Some(khz) = hypervisor_khz().filter(|khz| PLAUSIBLE_KHZ.contains(khz)) { + return (khz, TscSource::Hypervisor); + } + (calibrate_tsc_khz(), TscSource::Pit) +} + +fn cpuid_khz() -> Option { + if cpuid(0, 0).eax < 0x15 { + return None; + } + let leaf = cpuid(0x15, 0); + let (denominator, numerator, crystal_hz) = (leaf.eax, leaf.ebx, leaf.ecx); + (denominator != 0 && numerator != 0 && crystal_hz != 0) + .then(|| u64::from(crystal_hz) * u64::from(numerator) / u64::from(denominator) / 1000) +} + +fn hypervisor_khz() -> Option { + const KVM: [u32; 3] = [0x4b4d_564b, 0x564b_4d56, 0x0000_004d]; // "KVMKVMKVM\0\0\0" + const VMWARE: [u32; 3] = [0x6177_4d56, 0x4d56_6572, 0x6572_6177]; // "VMwareVMware" + if cpuid(1, 0).ecx & (1 << 31) == 0 { + return None; + } + let leaf = cpuid(0x4000_0000, 0); + let signature = [leaf.ebx, leaf.ecx, leaf.edx]; + if signature != KVM && signature != VMWARE || leaf.eax < 0x4000_0010 { + return None; + } + Some(u64::from(cpuid(0x4000_0010, 0).eax)) +} + +/// Uses PIT channel 2 exclusively for ~50 ms. +fn calibrate_tsc_khz() -> u64 { use x86_64::instructions::port::Port; const PORT_B: u16 = 0x61; // gate: bit 0, speaker: bit 1, OUT2: bit 5 diff --git a/litebox_hal/src/interrupt.rs b/litebox_hal/src/interrupt.rs index c3b75a0e9a..18ea81f691 100644 --- a/litebox_hal/src/interrupt.rs +++ b/litebox_hal/src/interrupt.rs @@ -6,7 +6,6 @@ //! xAPIC mode (e.g., QEMU's TCG); the legacy 8259 PICs are only masked. use crate::dma::{Hal, Mmio}; -use crate::pci::MsiMessage; use x86_64::instructions::port::Port; use x86_64::registers::model_specific::Msr; @@ -71,6 +70,13 @@ pub fn init_legacy_pics() { } } +/// An MSI message: a write of `data` to `address` raises the interrupt. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MsiMessage { + pub address: u64, + pub data: u32, +} + /// This CPU's local APIC, enabled, with its timer raising a vector when /// armed. pub struct LocalApic { diff --git a/litebox_hal/src/pci.rs b/litebox_hal/src/pci.rs index 9d400ee1a3..0b8ecb39fd 100644 --- a/litebox_hal/src/pci.rs +++ b/litebox_hal/src/pci.rs @@ -6,6 +6,7 @@ //! already assigned BARs. use crate::dma::Hal; +use crate::interrupt::MsiMessage; use x86_64::instructions::port::Port; const CONFIG_ADDRESS: u16 = 0xcf8; @@ -241,13 +242,6 @@ impl Function { } } -/// An MSI message: a write of `data` to `address` raises the interrupt. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct MsiMessage { - pub address: u64, - pub data: u32, -} - /// A memory BAR's physical range. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct MemoryBar { diff --git a/litebox_hal/src/virtio/console.rs b/litebox_hal/src/virtio/console.rs index 555c1e4dbf..5a287a8773 100644 --- a/litebox_hal/src/virtio/console.rs +++ b/litebox_hal/src/virtio/console.rs @@ -22,7 +22,7 @@ use super::Error; use super::pci::PciTransport; use super::queue::{Buffer, VirtQueue}; use crate::dma::{DmaRegion, Hal}; -use crate::pci::MsiMessage; +use crate::interrupt::MsiMessage; use alloc::collections::VecDeque; use alloc::vec; use alloc::vec::Vec; diff --git a/litebox_hal/src/virtio/pci.rs b/litebox_hal/src/virtio/pci.rs index bf190f5300..1f93b7c6be 100644 --- a/litebox_hal/src/virtio/pci.rs +++ b/litebox_hal/src/virtio/pci.rs @@ -7,7 +7,8 @@ use super::Error; use super::queue::VirtQueue; use crate::dma::{Hal, Mmio}; -use crate::pci::{self, Function, MemoryBar, MsiMessage}; +use crate::interrupt::MsiMessage; +use crate::pci::{self, Function, MemoryBar}; const CAP_VENDOR_SPECIFIC: u8 = 0x09; diff --git a/litebox_runner_vm_kernel/Cargo.toml b/litebox_runner_vm_kernel/Cargo.toml index 9e956bbc92..336723d896 100644 --- a/litebox_runner_vm_kernel/Cargo.toml +++ b/litebox_runner_vm_kernel/Cargo.toml @@ -8,8 +8,6 @@ description = "LiteBox as the guest kernel of a QEMU VM, hosting ring-3 runner p arrayvec = { version = "0.7.6", default-features = false } base64 = { version = "0.22.1", default-features = false, features = ["alloc"] } litebox_bootloader = { version = "0.1.0", path = "../litebox_bootloader" } -litebox_broker_core = { version = "0.1.0", path = "../litebox_broker_core" } -litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_vm_kernel = { version = "0.1.0", path = "../litebox_broker_vm_kernel" } litebox_hal = { version = "0.1.0", path = "../litebox_hal" } litebox_common_optee = { version = "0.1.0", path = "../litebox_common_optee" } diff --git a/litebox_runner_vm_kernel/src/devices.rs b/litebox_runner_vm_kernel/src/devices.rs index 8af3e9a902..a22928ab29 100644 --- a/litebox_runner_vm_kernel/src/devices.rs +++ b/litebox_runner_vm_kernel/src/devices.rs @@ -4,21 +4,17 @@ //! Devices the kernel drives, from `litebox_hal`, for the broker //! ([`litebox_broker_vm_kernel::Config`]): //! -//! - Runner processes' standard streams: a virtio console if there is one, -//! else output to the serial console and no input. -//! - Events: the console's interrupt (MSI-X), and the local APIC's timer +//! - [`Console`]: a virtio console if there is one, else output to the +//! serial console and no input. +//! - [`Events`]: the console's interrupt (MSI-X), and the local APIC's timer //! for deadlines. //! -//! The interrupt handler only ends the interrupt; the kernel services the -//! device when it next polls or halts. +//! The interrupt handler only ends the interrupt; the broker looks at the +//! devices when it next runs. use alloc::boxed::Box; -use core::sync::atomic::{AtomicBool, Ordering}; -use litebox_broker_core::readiness::{ReadinessRegistration, ReadinessWatchers}; -use litebox_broker_core::stdio::{ - StdioOutputStream, StdioProvider, StdioProviderError, StdioStream, -}; -use litebox_broker_protocol::readiness::ReadinessFlags; +use litebox_broker_vm_kernel::Events; +use litebox_broker_vm_kernel::stdio::Console; use litebox_hal::interrupt::LocalApic; use litebox_hal::virtio::console::VirtioConsole; use litebox_platform_vm_kernel::VmKernel; @@ -31,28 +27,13 @@ const WRITE_TIMEOUT_NANOS: u64 = 100_000_000; const TIMER_VECTOR: u8 = 0x30; const CONSOLE_VECTOR: u8 = 0x31; -struct Console { - device: Mutex, - /// Whether it interrupts; else it is polled. - interrupts: bool, - /// Standard-input files, to wake when input arrives. - stdin_watchers: Mutex, - /// Whether input was pending when last looked at; watchers are woken - /// when it starts to be. - stdin_readable: AtomicBool, -} - -impl Console { - fn readable(&self) -> bool { - self.device.lock().has_input() - } -} - pub struct Devices { platform: &'static VmKernel, apic: LocalApic, tsc_khz: u64, - console: Option, + console: Option>, + /// Whether the console interrupts; else it is polled. + console_interrupts: bool, } impl Devices { @@ -75,25 +56,22 @@ impl Devices { None } Some(Ok(device)) => { - let interrupts = device.transport().interrupts(); litebox_util_log::info!( pci:% = device.transport().function(), - interrupts:% = interrupts; + interrupts:% = device.transport().interrupts(); "virtio console: standard streams" ); - Some(Console { - device: Mutex::new(device), - interrupts, - stdin_watchers: Mutex::new(ReadinessWatchers::default()), - stdin_readable: AtomicBool::new(false), - }) + Some(device) } }; let devices: &'static Self = Box::leak(Box::new(Self { platform, apic, tsc_khz, - console, + console_interrupts: console + .as_ref() + .is_some_and(|device| device.transport().interrupts()), + console: console.map(Mutex::new), })); litebox_platform_vm_kernel::set_interrupt_handler(Box::leak(Box::new(|vector| { devices.on_interrupt(vector) @@ -128,95 +106,43 @@ fn rdtsc() -> u64 { unsafe { core::arch::x86_64::_rdtsc() } } -impl litebox_broker_vm_kernel::Events for Devices { - fn poll(&self) { - // Wake watchers when input starts to be pending. - if let Some(console) = &self.console { - let readable = console.readable(); - if readable && !console.stdin_readable.swap(true, Ordering::Relaxed) { - console.stdin_watchers.lock().publish(ReadinessFlags::READ); - } else if !readable { - console.stdin_readable.store(false, Ordering::Relaxed); - } - } - } - +impl Events for Devices { fn wait(&self, deadline: Option) -> bool { - let interrupts = self - .console - .as_ref() - .is_some_and(|console| console.interrupts); - if deadline.is_none() && !interrupts { + if deadline.is_none() && !self.console_interrupts { return false; } self.halt(deadline); - self.poll(); true } } -/// Standard streams over [`Devices`]. Never a terminal. Without a console, -/// standard input is at its end. -pub struct Stdio(pub &'static Devices); +/// Without a virtio console, input is at its end. +impl Console for Devices { + fn read(&self, output: &mut [u8]) -> Option { + Some(self.console.as_ref()?.lock().read(output)) + } -impl StdioProvider for Stdio { - fn read(&self, output: &mut [u8]) -> Result { - let Some(console) = &self.0.console else { - return Ok(0); - }; - match console.device.lock().read(output) { - 0 => Err(StdioProviderError::WouldBlock), - read => Ok(read), - } + fn has_input(&self) -> bool { + self.console + .as_ref() + .is_none_or(|device| device.lock().has_input()) } - /// Waits (bounded) for the console to take all of `input`. - fn write(&self, _stream: StdioOutputStream, input: &[u8]) -> Result { - let Some(console) = &self.0.console else { + /// Waits (bounded) for the device to take all of `input`. + fn write(&self, input: &[u8]) -> usize { + let Some(device) = &self.console else { litebox_hal::console::print_bytes(input); - return Ok(input.len()); + return input.len(); }; - let deadline = rdtsc() + WRITE_TIMEOUT_NANOS * self.0.tsc_khz / 1_000_000; + let deadline = rdtsc() + WRITE_TIMEOUT_NANOS * self.tsc_khz / 1_000_000; let mut written = 0; loop { - written += console.device.lock().write(&input[written..]); + written += device.lock().write(&input[written..]); if written == input.len() || rdtsc() >= deadline { - break; + return written; } // Until the device returns a transmit buffer. - self.0.halt(Some(deadline)); - } - if written == 0 { - return Err(StdioProviderError::WouldBlock); - } - Ok(written) - } - - fn is_terminal(&self, _stream: StdioStream) -> bool { - false - } - - fn readiness(&self, stream: StdioStream) -> ReadinessFlags { - match stream { - StdioStream::Stdin => match &self.0.console { - Some(console) if !console.readable() => ReadinessFlags(0), - _ => ReadinessFlags::READ, - }, - StdioStream::Stdout | StdioStream::Stderr => ReadinessFlags::WRITE, - } - } - - fn watch( - &self, - stream: StdioStream, - registration: &ReadinessRegistration, - ) -> litebox_broker_core::Result<()> { - if let (StdioStream::Stdin, Some(console)) = (stream, &self.0.console) { - console.stdin_watchers.lock().watch(registration)?; - if console.readable() { - registration.publish(ReadinessFlags::READ)?; - } + self.halt(Some(deadline)); } - Ok(()) } } diff --git a/litebox_runner_vm_kernel/src/main.rs b/litebox_runner_vm_kernel/src/main.rs index b11eeda1a0..1272996527 100644 --- a/litebox_runner_vm_kernel/src/main.rs +++ b/litebox_runner_vm_kernel/src/main.rs @@ -34,7 +34,7 @@ fn kernel_main(kernel: Kernel) -> ! { payload::Payload::read(kernel.boot_info).unwrap_or_else(|e| panic!("payload: {e}")); let devices = devices::Devices::init(kernel.platform, kernel.tsc_khz); let broker = litebox_broker_vm_kernel::Broker::new(litebox_broker_vm_kernel::Config { - stdio: alloc::sync::Arc::new(devices::Stdio(devices)), + console: devices, events: devices, }); let mut service = service::optee::Optee::new(kernel.platform, kernel.tsc_khz, &payload, broker); From 5b63e55a989573ce62f50ce8e2ef85d39c6be677 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Sat, 10 Oct 2026 02:33:16 +0000 Subject: [PATCH 4/7] separate device memory region --- Cargo.lock | 1 - .../src/arch/x86/mm/paging.rs | 38 ++++++++++++++----- litebox_platform_vm_kernel/src/dma.rs | 37 ++++++++---------- litebox_platform_vm_kernel/src/lib.rs | 26 ++++++++++--- litebox_shim_optee/Cargo.toml | 1 - 5 files changed, 64 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa3c8369ab..210ba07017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2199,7 +2199,6 @@ dependencies = [ "hashbrown", "hmac", "litebox", - "litebox_broker_protocol", "litebox_common_linux", "litebox_common_optee", "litebox_platform", diff --git a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs index b494607295..8b7f445fed 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/mm/paging.rs @@ -436,9 +436,32 @@ impl X64PageTable<'_, M, ALIGN> { Ok(()) } + /// Gives the kernel PML4 slot holding `va` its P3 table now, as Linux + /// preallocates its vmalloc area's: task tables share the kernel's slots + /// as they are at creation, so they see later mappings in it. + /// + /// # Panics + /// + /// Panics if `va` is below the kernel slots or the frame allocation fails. + pub(crate) fn preallocate_kernel_slot(&self, va: VirtAddr) { + assert!(usize::from(va.p4_index()) >= KERNEL_PML4_START); + let mut inner = self.inner.lock(); + let entry = &mut inner.level_4_table_mut()[va.p4_index()]; + if entry.is_unused() { + let frame = PageTableAllocator::::allocate_frame() + .expect("Failed to allocate a kernel P3 table"); + entry.set_frame( + frame, + PageTableFlags::PRESENT + | PageTableFlags::WRITABLE + | PageTableFlags::ACCESSED + | PageTableFlags::DIRTY, + ); + } + } + /// Maps device registers `frames` uncacheable at `start`, in the kernel - /// tables that task tables share. Pages already mapped to the same frame - /// are kept. + /// tables that task tables share. /// /// # Safety /// @@ -463,14 +486,11 @@ impl X64PageTable<'_, M, ALIGN> { let mut inner = self.inner.lock(); for (page, frame) in Page::range(start, start + frames.count() as u64).zip(frames) { // Safety: device registers (the caller's contract), mapped only here. - match unsafe { + unsafe { inner.map_to_with_table_flags(page, frame, flags, table_flags, &mut allocator) - } { - // A newly present entry needs no flush. - Ok(flush) => flush.ignore(), - Err(MapToError::PageAlreadyMapped(mapped)) if mapped == frame => {} - Err(error) => return Err(error), - } + }? + // A newly present entry needs no flush. + .ignore(); } Ok(()) } diff --git a/litebox_platform_vm_kernel/src/dma.rs b/litebox_platform_vm_kernel/src/dma.rs index d0aacd90b2..7363add5e2 100644 --- a/litebox_platform_vm_kernel/src/dma.rs +++ b/litebox_platform_vm_kernel/src/dma.rs @@ -3,23 +3,16 @@ //! [`litebox_hal::dma::Hal`]: DMA from the page allocator (RAM is //! direct-mapped, so contiguous), and device memory mapped uncacheable in -//! the kernel's PML4 slot, whose tables task page tables share, so they see -//! later mappings. Device memory below [`MMIO_WINDOW`] is mapped like RAM, -//! at `PA + KERNEL_OFFSET`; above it (firmware may place 64-bit BARs far -//! above RAM), in the window. +//! [`IOREMAP`], each mapping followed by an unmapped guard page, as Linux's +//! `ioremap` does. use crate::mm::MemoryProvider; -use crate::{KERNEL_OFFSET, VmKernel}; -use core::ops::Range; +use crate::{IOREMAP, VmKernel}; use core::ptr::NonNull; use litebox_hal::dma::{Hal, Mmio}; use x86_64::structures::paging::{Page, PageSize, PhysFrame, Size4KiB}; use x86_64::{PhysAddr, VirtAddr}; -/// The top of the kernel's PML4 slot, as offsets from `KERNEL_OFFSET`. RAM -/// must end below it. -pub(crate) const MMIO_WINDOW: Range = 448 << 30..512 << 30; - fn order(pages: usize) -> u32 { pages.next_power_of_two().trailing_zeros() } @@ -45,28 +38,28 @@ unsafe impl Hal for VmKernel { let first = PhysFrame::::containing_address(PhysAddr::try_new(pa).ok()?); let last = PhysFrame::containing_address(PhysAddr::try_new(end).ok()?.align_up(Size4KiB::SIZE)); - let ram = || self.ram_frame_ranges.iter(); - if ram().any(|r| r.start < last && first < r.end) - || ram().any(|r| r.end.start_address().as_u64() > MMIO_WINDOW.start) + if self + .ram_frame_ranges + .iter() + .any(|r| r.start < last && first < r.end) { return None; } - let start = if last.start_address().as_u64() <= MMIO_WINDOW.start { - first.start_address().as_u64() - } else { - let mut next = self.mmio_next.lock(); + let start = { + let mut next = self.ioremap_next.lock(); let start = *next; - let after = start.checked_add(last.start_address() - first.start_address())?; - if after > MMIO_WINDOW.end { + let after = start + .checked_add(last.start_address() - first.start_address())? + .checked_add(Size4KiB::SIZE)?; + if after > IOREMAP.end { return None; } *next = after; start }; - let page = Page::from_start_address(VirtAddr::new(KERNEL_OFFSET + start)).ok()?; + let page = Page::from_start_address(VirtAddr::new(start)).ok()?; // Safety: device registers (the caller's contract), not RAM (checked), - // at their direct-map address or a part of the window only this - // mapping gets. + // in a part of `IOREMAP` only this mapping gets. unsafe { self.page_table_manager .base_page_table diff --git a/litebox_platform_vm_kernel/src/lib.rs b/litebox_platform_vm_kernel/src/lib.rs index d6f581535d..281e074ad5 100644 --- a/litebox_platform_vm_kernel/src/lib.rs +++ b/litebox_platform_vm_kernel/src/lib.rs @@ -125,14 +125,21 @@ impl AddressSpaceId { pub const KERNEL: Self = Self(0); } -// Virtual address space: -// 0xFFFF_E200_0000_0000 .. kernel: all owned RAM at PA + KERNEL_OFFSET -// 0x0000_0000_0001_0000 .. 0x0000_7FFF_FFFF_F000 user +// Virtual address space, one 512 GiB PML4 slot per kernel region, as in +// Linux's Documentation/arch/x86/x86_64/mm.rst: +// 0x0000_0000_0001_0000 .. 0x0000_7FFF_FFFF_F000 user, per task +// 0xFFFF_E200_0000_0000 .. 0xFFFF_E280_0000_0000 direct map of all RAM, +// at PA + KERNEL_OFFSET +// 0xFFFF_E280_0000_0000 .. 0xFFFF_E300_0000_0000 ioremap: device memory /// `VA = PA + KERNEL_OFFSET` for kernel-owned RAM. Must be 512 GiB aligned /// (one PML4 slot). pub const KERNEL_OFFSET: u64 = 0xFFFF_E200_0000_0000; +/// Device memory mappings ([`litebox_hal::dma::Hal::map_mmio`]), the slot +/// after the direct map, which RAM must end below. +const IOREMAP: core::ops::Range = KERNEL_OFFSET + (512 << 30)..KERNEL_OFFSET + (1024 << 30); + /// Exclusive; the last page of the low canonical half is a guard page. const USER_ADDR_MAX: usize = 0x0000_7FFF_FFFF_F000; @@ -309,8 +316,8 @@ pub struct VmKernel { clock: &'static dyn clock::ClockSource, /// Guest RAM owned by this kernel (mapped at `PA + KERNEL_OFFSET`). ram_frame_ranges: alloc::vec::Vec>, - /// The next free offset in `dma::MMIO_WINDOW`. - mmio_next: spin::Mutex, + /// The next free address in [`IOREMAP`]. + ioremap_next: spin::Mutex, } /// Confines user pointers to the user range and opens SMAP around accesses. @@ -441,7 +448,14 @@ impl VmKernel { "kernel text and read-only data overlap" ); + assert!( + ram_frame_ranges + .iter() + .all(|r| KERNEL_OFFSET + r.end.start_address().as_u64() <= IOREMAP.start), + "guest RAM extends past the direct map" + ); let base_pt = mm::PageTable::new_top_level(); + base_pt.preallocate_kernel_slot(VirtAddr::new(IOREMAP.start)); for range in &ram_frame_ranges { if let Err(e) = base_pt.map_kernel_ram(*range, text, read_only) { panic!("failed to map guest RAM {range:?}: {e:?}"); @@ -460,7 +474,7 @@ impl VmKernel { page_table_manager: PageTableManager::new(base_pt), clock, ram_frame_ranges, - mmio_next: spin::Mutex::new(dma::MMIO_WINDOW.start), + ioremap_next: spin::Mutex::new(IOREMAP.start), })) } diff --git a/litebox_shim_optee/Cargo.toml b/litebox_shim_optee/Cargo.toml index 25ef836166..c3f9325c31 100644 --- a/litebox_shim_optee/Cargo.toml +++ b/litebox_shim_optee/Cargo.toml @@ -9,7 +9,6 @@ ctr = { version = "0.8", default-features = false } elf = { version = "0.8.0", default-features = false } hashbrown = "0.15.2" litebox = { path = "../litebox/", version = "0.1.0" } -litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } litebox_common_linux = { path = "../litebox_common_linux/", version = "0.1.0" } litebox_common_optee = { path = "../litebox_common_optee/", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } From e152c91cf771d06290026495298ba203ec50ae2c Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Mon, 12 Oct 2026 02:22:45 +0000 Subject: [PATCH 5/7] further abstract clock, console, and interrupt --- litebox_bootloader/src/lib.rs | 24 ++--- litebox_broker_vm_kernel/src/lib.rs | 11 +- litebox_hal/src/clock.rs | 73 ++++++++++--- litebox_hal/src/devices.rs | 102 ++++++++++++++++++ litebox_hal/src/interrupt.rs | 62 ++++++----- litebox_hal/src/lib.rs | 8 +- litebox_hal/src/{console.rs => serial.rs} | 11 +- litebox_platform_vm_kernel/src/clock.rs | 21 +--- litebox_runner_optee_on_qemu/src/main.rs | 8 +- litebox_runner_vm_kernel/src/devices.rs | 93 ++++++---------- litebox_runner_vm_kernel/src/main.rs | 16 +-- litebox_runner_vm_kernel/src/service/optee.rs | 5 +- 12 files changed, 276 insertions(+), 158 deletions(-) create mode 100644 litebox_hal/src/devices.rs rename litebox_hal/src/{console.rs => serial.rs} (91%) diff --git a/litebox_bootloader/src/lib.rs b/litebox_bootloader/src/lib.rs index b3450504d5..917b3e2a21 100644 --- a/litebox_bootloader/src/lib.rs +++ b/litebox_bootloader/src/lib.rs @@ -33,7 +33,8 @@ use layout::{ heap_start_address, rodata_end_address, rodata_start_address, text_end_address, text_start_address, }; -use litebox_platform_vm_kernel::{BootConfig, KERNEL_OFFSET, VmKernel, clock::TscClock}; +use litebox_hal::clock::Clock; +use litebox_platform_vm_kernel::{BootConfig, KERNEL_OFFSET, VmKernel}; use x86_64::PhysAddr; const PAGE_SIZE: u64 = 4096; @@ -43,7 +44,7 @@ pub struct Kernel { /// Boot modules stay mapped at `PA + KERNEL_OFFSET`, in kernel-managed /// RAM, for the kernel's lifetime. pub boot_info: &'static BootInfo, - pub tsc_khz: u64, + pub clock: Clock, } /// Declares the kernel's `fn(Kernel) -> !`, run on the kernel stack once the @@ -70,15 +71,15 @@ unsafe extern "Rust" { safe fn __litebox_kernel_logger() -> &'static dyn log::Log; } -static BOOT: spin::Once<(BootInfo, u64)> = spin::Once::new(); +static BOOT: spin::Once<(BootInfo, Clock)> = spin::Once::new(); fn early_console_init() { let _ = log::set_logger(__litebox_kernel_logger()); log::set_max_level(log::LevelFilter::Info); - litebox_hal::console_println!("====================="); - litebox_hal::console_println!(" Hello from LiteBox! "); - litebox_hal::console_println!("====================="); + litebox_hal::serial_println!("====================="); + litebox_hal::serial_println!(" Hello from LiteBox! "); + litebox_hal::serial_println!("====================="); } /// The front ends' entry; see [`handoff`]. @@ -101,10 +102,9 @@ fn kernel_start(boot_info: impl FnOnce() -> BootInfo) -> ! { .collect(); let to_pa = |va: u64| VmKernel::va_to_pa(x86_64::VirtAddr::new(va)); litebox_hal::interrupt::init_legacy_pics(); - let (tsc_khz, source) = litebox_hal::clock::tsc_khz(); - litebox_util_log::info!(mhz:% = tsc_khz / 1000, source:? = source; "TSC frequency"); - BOOT.call_once(|| (info, tsc_khz)); - let clock = alloc::boxed::Box::leak(alloc::boxed::Box::new(TscClock::new(tsc_khz))); + let clock = Clock::measure(); + BOOT.call_once(|| (info, clock)); + let clock = alloc::boxed::Box::leak(alloc::boxed::Box::new(clock)); // Safety: the front end established the direct mapping with IRQs off on one // CPU. The heap contains only `ram`, text bounds come from the linker, and // no live resource needs the boot stack after the handoff. @@ -124,7 +124,7 @@ fn kernel_start(boot_info: impl FnOnce() -> BootInfo) -> ! { } fn kernel_main(platform: &'static VmKernel) -> ! { - let (boot_info, tsc_khz) = BOOT.get().expect("boot info captured"); + let (boot_info, clock) = BOOT.get().expect("boot info captured"); for m in &boot_info.modules { assert!( platform.contains_ram(m.clone()), @@ -136,7 +136,7 @@ fn kernel_main(platform: &'static VmKernel) -> ! { __litebox_kernel_main(Kernel { platform, boot_info, - tsc_khz: *tsc_khz, + clock: *clock, }) } diff --git a/litebox_broker_vm_kernel/src/lib.rs b/litebox_broker_vm_kernel/src/lib.rs index f4f1de4f60..b49579ec25 100644 --- a/litebox_broker_vm_kernel/src/lib.rs +++ b/litebox_broker_vm_kernel/src/lib.rs @@ -58,9 +58,10 @@ pub const CONTROL_RING_SIZE: usize = CONTROL_RING_MEMORY_SIZE; /// What the broker waits for: the runner's event sources (e.g., device /// interrupts and a timer). pub trait Events: Sync { - /// Halts until an event may have happened, or until `deadline` (TSC) - /// passes. Without a deadline, false if no event could ever end the halt - /// (and it does not halt). + /// Halts until an event may have happened, or until `deadline` (a TSC + /// value, from ring 3; see [`litebox_hal::clock::now`]) passes. Without a + /// deadline, false if no event could ever end the halt (and it does not + /// halt). fn wait(&self, deadline: Option) -> bool; } @@ -309,9 +310,7 @@ impl Association { Ok(_) => {} Err(_) => return Err(invalid()), } - // Safety: RDTSC has no side effects. - if deadline.is_some_and(|deadline| unsafe { core::arch::x86_64::_rdtsc() } >= deadline) - { + if deadline.is_some_and(|deadline| litebox_hal::clock::now() >= deadline) { return Err(EnterError::Status(Status::TimedOut)); } if deadline.is_none() && !asynchronous { diff --git a/litebox_hal/src/clock.rs b/litebox_hal/src/clock.rs index 014986cc73..ed86e23095 100644 --- a/litebox_hal/src/clock.rs +++ b/litebox_hal/src/clock.rs @@ -1,11 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Clocks: the TSC's frequency, from the most authoritative source the -//! machine has: CPUID, the hypervisor, or else calibration against the -//! legacy PIT (which, e.g., Firecracker and Cloud Hypervisor lack). +//! The CPU's clock: a free-running counter ([`now`], in ticks) and its +//! frequency ([`Clock`]). On x86-64, the counter is the TSC, which ring 3 +//! reads too: `BrokerEnter` deadlines are TSC values and +//! `StartupInfo::tsc_khz` is [`Clock::khz`] (see `litebox_common_vm_abi`), so +//! the TSC is part of the ABI. Its frequency comes from the most +//! authoritative source the machine has: CPUID, the hypervisor, or else +//! calibration against the legacy PIT (which, e.g., Firecracker and Cloud +//! Hypervisor lack). use core::arch::x86_64::__cpuid_count as cpuid; +use core::time::Duration; const PIT_HZ: u64 = 1_193_182; const CALIBRATION_MS: u64 = 50; @@ -14,9 +20,52 @@ const CALIBRATION_TIMEOUT_CYCLES: u64 = 1 << 36; /// Frequencies outside this range are not believed. const PLAUSIBLE_KHZ: core::ops::RangeInclusive = 100_000..=100_000_000; +/// The counter: the TSC, as ring 3 reads it (see the module docs). Never +/// decreases (one CPU, constant rate). +pub fn now() -> u64 { + // Safety: RDTSC has no side effects. + unsafe { core::arch::x86_64::_rdtsc() } +} + +/// The counter's frequency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Clock { + khz: u64, +} + +impl Clock { + /// Finds the frequency (logged, with its source). Call once, at boot; + /// calibration takes a while. + /// + /// # Panics + /// + /// If no source works. + pub fn measure() -> Self { + let (khz, source) = tsc_khz(); + litebox_util_log::info!(mhz:% = khz / 1000, source:? = source; "clock frequency"); + Self { khz } + } + + /// Ticks per millisecond: the TSC frequency in kHz, as ring 3 gets it. + pub fn khz(self) -> u64 { + self.khz + } + + /// [`now`] in nanoseconds. + pub fn nanos(self) -> u64 { + u64::try_from(u128::from(now()) * 1_000_000 / u128::from(self.khz)).unwrap_or(u64::MAX) + } + + /// The tick [`now`] reaches `timeout` from now. + pub fn deadline(self, timeout: Duration) -> u64 { + let ticks = timeout.as_nanos() * u128::from(self.khz) / 1_000_000; + now().saturating_add(u64::try_from(ticks).unwrap_or(u64::MAX)) + } +} + /// Where [`tsc_khz`] got the frequency. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum TscSource { +enum TscSource { /// CPUID leaf 0x15: the core crystal and its ratio (Intel). Cpuid, /// The hypervisor's timing leaf 0x40000010 (KVM via QEMU, VMware). @@ -26,11 +75,7 @@ pub enum TscSource { } /// The TSC frequency in kHz, and its source. -/// -/// # Panics -/// -/// If no source works. -pub fn tsc_khz() -> (u64, TscSource) { +fn tsc_khz() -> (u64, TscSource) { if let Some(khz) = cpuid_khz().filter(|khz| PLAUSIBLE_KHZ.contains(khz)) { return (khz, TscSource::Cpuid); } @@ -85,18 +130,18 @@ fn calibrate_tsc_khz() -> u64 { ch2.write(hi); // Mode 0 starts counting on the PIT clock after the MSB write. - let start = core::arch::x86_64::_rdtsc(); - let mut now = start; + let start = now(); + let mut end = start; while port_b.read() & 0x20 == 0 { - now = core::arch::x86_64::_rdtsc(); + end = now(); assert!( - now - start < CALIBRATION_TIMEOUT_CYCLES, + end - start < CALIBRATION_TIMEOUT_CYCLES, "TSC calibration: the legacy PIT (channel 2) never signalled" ); } // Bits 4-7 are read-only. port_b.write(v & 0x0f); - let khz = (now - start) / CALIBRATION_MS; + let khz = (end - start) / CALIBRATION_MS; assert!(khz != 0, "TSC calibration measured a zero frequency"); khz } diff --git a/litebox_hal/src/devices.rs b/litebox_hal/src/devices.rs new file mode 100644 index 0000000000..6373641c93 --- /dev/null +++ b/litebox_hal/src/devices.rs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Devices behind device-neutral types ([`ConsoleDevice`]), each probing +//! the drivers the HAL has for its kind. Runners choose which devices to +//! probe, where they interrupt, and what to do without one. + +use crate::dma::Hal; +use crate::interrupt::InterruptTarget; +use crate::virtio::console::VirtioConsole; +use alloc::boxed::Box; +use core::fmt; + +/// A console: output, and input if the device receives any. Not internally +/// synchronized. +pub struct ConsoleDevice(Backend); + +enum Backend { + Virtio(Box), + /// The kernel's UART ([`crate::serial`]): output only. + Serial, +} + +impl ConsoleDevice { + /// The VM's console device, interrupting `interrupt` if it can (see + /// [`Self::interrupts`]); `None` if there is none. + /// + /// # Errors + /// + /// The device cannot be driven. + pub fn probe( + hal: &'static dyn Hal, + interrupt: InterruptTarget, + ) -> Result, crate::virtio::Error> { + // Virtio over PCI. + let device = VirtioConsole::probe(hal, interrupt.msi_message()).transpose()?; + Ok(device.map(|device| Self(Backend::Virtio(Box::new(device))))) + } + + /// Output to the kernel's UART; receives nothing. + pub fn serial() -> Self { + Self(Backend::Serial) + } + + /// Whether the device receives input at all. + pub fn receives(&self) -> bool { + match &self.0 { + Backend::Virtio(_) => true, + Backend::Serial => false, + } + } + + /// Whether the device raises its interrupt; else it must be polled. + pub fn interrupts(&self) -> bool { + match &self.0 { + Backend::Virtio(device) => device.transport().interrupts(), + Backend::Serial => false, + } + } + + /// Queues as much of `bytes` as the device takes now; returns how much. + pub fn write(&mut self, bytes: &[u8]) -> usize { + match &mut self.0 { + Backend::Virtio(device) => device.write(bytes), + Backend::Serial => { + crate::serial::print_bytes(bytes); + bytes.len() + } + } + } + + /// Whether received input is waiting. + pub fn has_input(&mut self) -> bool { + match &mut self.0 { + Backend::Virtio(device) => device.has_input(), + Backend::Serial => false, + } + } + + /// Copies received input into `output`; returns how much (zero if none + /// is waiting). + pub fn read(&mut self, output: &mut [u8]) -> usize { + match &mut self.0 { + Backend::Virtio(device) => device.read(output), + Backend::Serial => 0, + } + } +} + +impl fmt::Display for ConsoleDevice { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.0 { + Backend::Virtio(device) => write!( + f, + "virtio (pci {}, interrupts {})", + device.transport().function(), + device.transport().interrupts() + ), + Backend::Serial => f.write_str("serial (output only)"), + } + } +} diff --git a/litebox_hal/src/interrupt.rs b/litebox_hal/src/interrupt.rs index 18ea81f691..66d9531ac7 100644 --- a/litebox_hal/src/interrupt.rs +++ b/litebox_hal/src/interrupt.rs @@ -5,6 +5,7 @@ //! the timer, in x2APIC mode (as on LVBS) where the CPU has it, else in //! xAPIC mode (e.g., QEMU's TCG); the legacy 8259 PICs are only masked. +use crate::clock::{Clock, now}; use crate::dma::{Hal, Mmio}; use x86_64::instructions::port::Port; use x86_64::registers::model_specific::Msr; @@ -77,6 +78,28 @@ pub struct MsiMessage { pub data: u32, } +/// Where a device interrupt goes: `vector` on the CPU with local APIC ID +/// `apic_id`. Each delivery mechanism derives its own form (e.g., an MSI +/// message). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InterruptTarget { + pub apic_id: u32, + pub vector: u8, +} + +impl InterruptTarget { + /// As an MSI message (fixed delivery, edge); `None` if the APIC ID does + /// not fit an MSI address, which without interrupt remapping holds 8 + /// bits. + pub fn msi_message(self) -> Option { + let id = u8::try_from(self.apic_id).ok()?; + Some(MsiMessage { + address: 0xfee0_0000 | (u64::from(id) << 12), + data: u32::from(self.vector), + }) + } +} + /// This CPU's local APIC, enabled, with its timer raising a vector when /// armed. pub struct LocalApic { @@ -91,20 +114,15 @@ fn has_cpu_feature(ecx_bit: u32) -> bool { core::arch::x86_64::__cpuid_count(1, 0).ecx & (1 << ecx_bit) != 0 } -fn rdtsc() -> u64 { - // Safety: RDTSC has no side effects. - unsafe { core::arch::x86_64::_rdtsc() } -} - impl LocalApic { /// Enables the local APIC and sets its timer to raise `timer_vector`, - /// disarmed. Calibrates the timer against the TSC (`tsc_khz`) if it has - /// no TSC-deadline mode. + /// disarmed. Calibrates the timer against `clock` if it has no + /// TSC-deadline mode. /// /// # Panics /// /// If the xAPIC registers cannot be mapped. - pub fn init(hal: &dyn Hal, timer_vector: u8, tsc_khz: u64) -> Self { + pub fn init(hal: &dyn Hal, timer_vector: u8, clock: Clock) -> Self { // The local APIC belongs to this kernel, and interrupts are disabled. // The SDM requires enabling xAPIC before x2APIC. let mut base = Msr::new(IA32_APIC_BASE); @@ -139,9 +157,9 @@ impl LocalApic { apic.write(reg::TIMER_DIVIDE, TIMER_DIVIDE_BY_16); apic.write(reg::LVT_TIMER, LVT_TIMER_MASKED | u32::from(timer_vector)); apic.write(reg::TIMER_INITIAL_COUNT, u32::MAX); - let start = rdtsc(); - let cycles = TIMER_CALIBRATION_MS * tsc_khz; - while rdtsc() - start < cycles {} + let start = now(); + let cycles = TIMER_CALIBRATION_MS * clock.khz(); + while now() - start < cycles {} let ticks = u64::from(u32::MAX - apic.read(reg::TIMER_CURRENT_COUNT)); apic.write(reg::TIMER_INITIAL_COUNT, 0); apic.write(reg::LVT_TIMER, u32::from(timer_vector)); @@ -183,26 +201,16 @@ impl LocalApic { self.write(reg::EOI, 0); } - /// The MSI message that raises `vector` on this CPU (fixed delivery, - /// edge). - /// - /// # Panics - /// - /// If the APIC ID does not fit an MSI address (above 255). - pub fn msi_message(&self, vector: u8) -> MsiMessage { - let id = match self.xapic { + /// Interrupts raising `vector` on this CPU. + pub fn target(&self, vector: u8) -> InterruptTarget { + let apic_id = match self.xapic { Some(_) => self.read(reg::ID) >> 24, None => self.read(reg::ID), }; - // Without interrupt remapping, MSI addresses an 8-bit APIC ID. - assert!(id <= 0xff, "APIC ID {id} beyond MSI's reach"); - MsiMessage { - address: 0xfee0_0000 | (u64::from(id) << 12), - data: u32::from(vector), - } + InterruptTarget { apic_id, vector } } - /// Raises the timer's vector once, when the TSC reaches `deadline` (at + /// Raises the timer's vector once, when [`now`] reaches `deadline` (at /// once if it has; without TSC-deadline mode, a distant deadline may come /// early); replaces a pending deadline. pub fn arm_timer(&self, deadline: u64) { @@ -210,7 +218,7 @@ impl LocalApic { // Safety: programs this CPU's timer; zero would disarm. None => unsafe { Msr::new(IA32_TSC_DEADLINE).write(deadline.max(1)) }, Some((ticks, cycles)) => { - let delta = u128::from(deadline.saturating_sub(rdtsc())); + let delta = u128::from(deadline.saturating_sub(now())); let count = delta * u128::from(ticks) / u128::from(cycles); self.write( reg::TIMER_INITIAL_COUNT, diff --git a/litebox_hal/src/lib.rs b/litebox_hal/src/lib.rs index df93381525..a153d674da 100644 --- a/litebox_hal/src/lib.rs +++ b/litebox_hal/src/lib.rs @@ -3,8 +3,8 @@ //! Hardware abstraction layer for LiteBox kernel-mode runners: what a VM //! cannot do by itself and gets from its (virtual) hardware or hypervisor. -//! Each module states the hardware it assumes. Runners choose and wire the -//! sources. +//! Each module states the hardware it assumes. [`devices`] puts devices +//! behind device-neutral types; runners choose and wire the sources. #![no_std] #![warn(clippy::undocumented_unsafe_blocks)] @@ -14,7 +14,7 @@ extern crate alloc; #[cfg(target_arch = "x86_64")] pub mod clock; #[cfg(target_arch = "x86_64")] -pub mod console; +pub mod devices; pub mod dma; #[cfg(target_arch = "x86_64")] pub mod interrupt; @@ -26,4 +26,6 @@ pub mod prk; #[cfg(target_arch = "x86_64")] pub mod rng; #[cfg(target_arch = "x86_64")] +pub mod serial; +#[cfg(target_arch = "x86_64")] pub mod virtio; diff --git a/litebox_hal/src/console.rs b/litebox_hal/src/serial.rs similarity index 91% rename from litebox_hal/src/console.rs rename to litebox_hal/src/serial.rs index f5081981ba..65afd02d0e 100644 --- a/litebox_hal/src/console.rs +++ b/litebox_hal/src/serial.rs @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Console: a 16550 UART on COM1. Output is dropped if no UART answers. +//! The serial port, for the kernel's own output: a 16550 UART on COM1. +//! Output is dropped if no UART answers. use core::fmt; use spin::{Mutex, Once}; @@ -138,9 +139,9 @@ pub fn panic_print(args: core::fmt::Arguments) { let _ = com.write_fmt(args); } -/// `println!` to the console. +/// `println!` to the serial port. #[macro_export] -macro_rules! console_println { - () => ($crate::console::print(format_args!("\n"))); - ($($arg:tt)*) => ($crate::console::print(format_args!("{}\n", format_args!($($arg)*)))); +macro_rules! serial_println { + () => ($crate::serial::print(format_args!("\n"))); + ($($arg:tt)*) => ($crate::serial::print(format_args!("{}\n", format_args!($($arg)*)))); } diff --git a/litebox_platform_vm_kernel/src/clock.rs b/litebox_platform_vm_kernel/src/clock.rs index a607fc5194..1f1132ee3c 100644 --- a/litebox_platform_vm_kernel/src/clock.rs +++ b/litebox_platform_vm_kernel/src/clock.rs @@ -9,25 +9,8 @@ pub trait ClockSource: Sync { fn monotonic_nanos(&self) -> u64; } -/// Requires a single CPU and a constant TSC rate; neither is checked. -pub struct TscClock { - khz: u64, -} - -impl TscClock { - /// # Panics - /// - /// Panics if `khz` is zero. - pub fn new(khz: u64) -> Self { - assert!(khz != 0, "TSC frequency must not be zero"); - Self { khz } - } -} - -impl ClockSource for TscClock { +impl ClockSource for litebox_hal::clock::Clock { fn monotonic_nanos(&self) -> u64 { - // Safety: RDTSC has no side effects. - let tsc = u128::from(unsafe { core::arch::x86_64::_rdtsc() }); - u64::try_from(tsc * 1_000_000 / u128::from(self.khz)).unwrap_or(u64::MAX) + self.nanos() } } diff --git a/litebox_runner_optee_on_qemu/src/main.rs b/litebox_runner_optee_on_qemu/src/main.rs index 2adcfb2dce..d1c4436a53 100644 --- a/litebox_runner_optee_on_qemu/src/main.rs +++ b/litebox_runner_optee_on_qemu/src/main.rs @@ -18,7 +18,7 @@ mod tests; use core::panic::PanicInfo; use litebox_bootloader::Kernel; -use litebox_hal::{console, console_println, power}; +use litebox_hal::{power, serial, serial_println}; use litebox_platform_vm_kernel::providers::set_platform_root_key; litebox_bootloader::entry!(kernel_main, &SERIAL_LOGGER); @@ -28,7 +28,7 @@ fn kernel_main(kernel: Kernel) -> ! { tests::check_user_memory_protection(kernel.platform); optee::run(kernel.platform, kernel.boot_info); - console_println!("[litebox] ALL TESTS PASSED"); + serial_println!("[litebox] ALL TESTS PASSED"); power::exit(true) } @@ -45,7 +45,7 @@ impl log::Log for SerialLogger { } let mut buf: arrayvec::ArrayString<1024> = arrayvec::ArrayString::new(); let _ = litebox_util_log::format_record(&mut buf, record); - console::print_str(&buf); + serial::print_str(&buf); } fn flush(&self) {} @@ -69,7 +69,7 @@ static SERIAL_LOGGER: SerialLogger = SerialLogger; #[panic_handler] fn panic(info: &PanicInfo) -> ! { - console::panic_print(format_args!( + serial::panic_print(format_args!( "[litebox] PANIC: {info}\n[litebox] TESTS FAILED\n" )); power::exit(false) diff --git a/litebox_runner_vm_kernel/src/devices.rs b/litebox_runner_vm_kernel/src/devices.rs index a22928ab29..475f4aa63b 100644 --- a/litebox_runner_vm_kernel/src/devices.rs +++ b/litebox_runner_vm_kernel/src/devices.rs @@ -4,74 +4,60 @@ //! Devices the kernel drives, from `litebox_hal`, for the broker //! ([`litebox_broker_vm_kernel::Config`]): //! -//! - [`Console`]: a virtio console if there is one, else output to the -//! serial console and no input. -//! - [`Events`]: the console's interrupt (MSI-X), and the local APIC's timer -//! for deadlines. +//! - [`Console`]: the VM's console device, else output to the serial port +//! and no input. +//! - [`Events`]: the devices' interrupt, and the local APIC's timer for +//! deadlines. //! //! The interrupt handler only ends the interrupt; the broker looks at the //! devices when it next runs. use alloc::boxed::Box; +use core::time::Duration; use litebox_broker_vm_kernel::Events; use litebox_broker_vm_kernel::stdio::Console; +use litebox_hal::clock::{self, Clock}; +use litebox_hal::devices::ConsoleDevice; use litebox_hal::interrupt::LocalApic; -use litebox_hal::virtio::console::VirtioConsole; use litebox_platform_vm_kernel::VmKernel; use spin::Mutex; /// How long a console write may wait for the device to take output before /// the rest is dropped. -const WRITE_TIMEOUT_NANOS: u64 = 100_000_000; +const WRITE_TIMEOUT: Duration = Duration::from_millis(100); const TIMER_VECTOR: u8 = 0x30; -const CONSOLE_VECTOR: u8 = 0x31; +/// Shared by every device: the broker looks at all of them when it runs. +const DEVICE_VECTOR: u8 = 0x31; pub struct Devices { platform: &'static VmKernel, apic: LocalApic, - tsc_khz: u64, - console: Option>, + clock: Clock, + console: Mutex, /// Whether the console interrupts; else it is polled. console_interrupts: bool, } impl Devices { /// Brings up the devices and enables their interrupts. Call once. - pub fn init(platform: &'static VmKernel, tsc_khz: u64) -> &'static Self { - let apic = LocalApic::init(platform, TIMER_VECTOR, tsc_khz); - let interrupt = apic.msi_message(CONSOLE_VECTOR); - let console = match VirtioConsole::probe(platform, Some(interrupt)) { - None => { - litebox_util_log::info!( - "no virtio console: standard output goes to the serial console" - ); - None - } - Some(Err(error)) => { - litebox_util_log::warn!( - error:% = error; - "virtio console unusable: standard output goes to the serial console" - ); - None - } - Some(Ok(device)) => { - litebox_util_log::info!( - pci:% = device.transport().function(), - interrupts:% = device.transport().interrupts(); - "virtio console: standard streams" - ); - Some(device) + pub fn init(platform: &'static VmKernel, clock: Clock) -> &'static Self { + let apic = LocalApic::init(platform, TIMER_VECTOR, clock); + let console = match ConsoleDevice::probe(platform, apic.target(DEVICE_VECTOR)) { + Ok(Some(device)) => device, + Ok(None) => ConsoleDevice::serial(), + Err(error) => { + litebox_util_log::warn!(error:% = error; "console device unusable"); + ConsoleDevice::serial() } }; + litebox_util_log::info!(device:% = console; "standard streams"); let devices: &'static Self = Box::leak(Box::new(Self { platform, apic, - tsc_khz, - console_interrupts: console - .as_ref() - .is_some_and(|device| device.transport().interrupts()), - console: console.map(Mutex::new), + clock, + console_interrupts: console.interrupts(), + console: Mutex::new(console), })); litebox_platform_vm_kernel::set_interrupt_handler(Box::leak(Box::new(|vector| { devices.on_interrupt(vector) @@ -82,14 +68,14 @@ impl Devices { /// See `litebox_platform_vm_kernel::set_interrupt_handler`: no locks, no /// per-CPU data. fn on_interrupt(&self, vector: u8) -> bool { - if !matches!(vector, TIMER_VECTOR | CONSOLE_VECTOR) { + if !matches!(vector, TIMER_VECTOR | DEVICE_VECTOR) { return false; } self.apic.end_of_interrupt(); true } - /// Halts until an interrupt, or until `deadline` (TSC). + /// Halts until an interrupt, or until `deadline` (a [`clock::now`] value). fn halt(&self, deadline: Option) { if let Some(deadline) = deadline { self.apic.arm_timer(deadline); @@ -101,11 +87,6 @@ impl Devices { } } -fn rdtsc() -> u64 { - // Safety: RDTSC has no side effects. - unsafe { core::arch::x86_64::_rdtsc() } -} - impl Events for Devices { fn wait(&self, deadline: Option) -> bool { if deadline.is_none() && !self.console_interrupts { @@ -116,32 +97,28 @@ impl Events for Devices { } } -/// Without a virtio console, input is at its end. +/// A console that receives nothing is at the end of its input. impl Console for Devices { fn read(&self, output: &mut [u8]) -> Option { - Some(self.console.as_ref()?.lock().read(output)) + let mut console = self.console.lock(); + console.receives().then(|| console.read(output)) } fn has_input(&self) -> bool { - self.console - .as_ref() - .is_none_or(|device| device.lock().has_input()) + let mut console = self.console.lock(); + !console.receives() || console.has_input() } /// Waits (bounded) for the device to take all of `input`. fn write(&self, input: &[u8]) -> usize { - let Some(device) = &self.console else { - litebox_hal::console::print_bytes(input); - return input.len(); - }; - let deadline = rdtsc() + WRITE_TIMEOUT_NANOS * self.tsc_khz / 1_000_000; + let deadline = self.clock.deadline(WRITE_TIMEOUT); let mut written = 0; loop { - written += device.lock().write(&input[written..]); - if written == input.len() || rdtsc() >= deadline { + written += self.console.lock().write(&input[written..]); + if written == input.len() || clock::now() >= deadline { return written; } - // Until the device returns a transmit buffer. + // Until the device takes more output. self.halt(Some(deadline)); } } diff --git a/litebox_runner_vm_kernel/src/main.rs b/litebox_runner_vm_kernel/src/main.rs index 1272996527..75a2108261 100644 --- a/litebox_runner_vm_kernel/src/main.rs +++ b/litebox_runner_vm_kernel/src/main.rs @@ -6,8 +6,8 @@ //! `litebox_runner_optee_on_vm_userland` processes, one per TA instance //! ([`service::optee`]), //! driven by a test client ([`client::script`]). QEMU exits with 33 if all -//! tests pass, 65 otherwise. Runner processes' standard streams are a virtio -//! console, if there is one ([`devices`]). +//! tests pass, 65 otherwise. Runner processes' standard streams are the +//! VM's console device, if there is one ([`devices`]). #![cfg(target_arch = "x86_64")] #![no_std] @@ -23,7 +23,7 @@ mod service; use core::panic::PanicInfo; use litebox_bootloader::Kernel; -use litebox_hal::{console, console_println, power}; +use litebox_hal::{power, serial, serial_println}; use litebox_platform_vm_kernel::providers::set_platform_root_key; litebox_bootloader::entry!(kernel_main, &SerialLogger); @@ -32,14 +32,14 @@ fn kernel_main(kernel: Kernel) -> ! { set_platform_root_key(&litebox_hal::prk::development()); let payload = payload::Payload::read(kernel.boot_info).unwrap_or_else(|e| panic!("payload: {e}")); - let devices = devices::Devices::init(kernel.platform, kernel.tsc_khz); + let devices = devices::Devices::init(kernel.platform, kernel.clock); let broker = litebox_broker_vm_kernel::Broker::new(litebox_broker_vm_kernel::Config { console: devices, events: devices, }); - let mut service = service::optee::Optee::new(kernel.platform, kernel.tsc_khz, &payload, broker); + let mut service = service::optee::Optee::new(kernel.platform, kernel.clock, &payload, broker); client::script::run(&payload, &mut service); - console_println!("[litebox] ALL TESTS PASSED"); + serial_println!("[litebox] ALL TESTS PASSED"); power::exit(true) } @@ -56,7 +56,7 @@ impl log::Log for SerialLogger { } let mut buf: arrayvec::ArrayString<1024> = arrayvec::ArrayString::new(); let _ = litebox_util_log::format_record(&mut buf, record); - console::print_str(&buf); + serial::print_str(&buf); } fn flush(&self) {} @@ -78,7 +78,7 @@ fn is_spurious_tar_error(record: &log::Record) -> bool { #[panic_handler] fn panic(info: &PanicInfo) -> ! { - console::panic_print(format_args!( + serial::panic_print(format_args!( "[litebox] PANIC: {info}\n[litebox] TESTS FAILED\n" )); power::exit(false) diff --git a/litebox_runner_vm_kernel/src/service/optee.rs b/litebox_runner_vm_kernel/src/service/optee.rs index ee04a48275..8a33f6c6df 100644 --- a/litebox_runner_vm_kernel/src/service/optee.rs +++ b/litebox_runner_vm_kernel/src/service/optee.rs @@ -13,6 +13,7 @@ use alloc::collections::BTreeMap; use litebox_common_optee::TeeUuid; use litebox_common_optee::envelope::{LDELF_IMAGE, TA_IMAGE}; use litebox_common_vm_abi::IDENTITY_LEN; +use litebox_hal::clock::Clock; use litebox_platform_vm_kernel::VmKernel; use litebox_shim_vm_kernel::optee::Completion; use litebox_shim_vm_kernel::optee::Invocation; @@ -38,7 +39,7 @@ impl Optee { /// On a missing or malformed payload file. pub fn new( platform: &'static VmKernel, - tsc_khz: u64, + clock: Clock, payload: &Payload, broker: litebox_broker_vm_kernel::Broker, ) -> Self { @@ -88,7 +89,7 @@ impl Optee { runner, images: &images, identity, - tsc_khz, + tsc_khz: clock.khz(), }, ) .map_err(|error| { From 89c90c44347a941709d9918dd8f1775bdf2200a5 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Mon, 12 Oct 2026 05:30:04 +0000 Subject: [PATCH 6/7] fix --- litebox_hal/src/virtio/pci.rs | 4 +++- litebox_platform_vm_kernel/src/arch/x86/interrupts.rs | 5 +++-- litebox_platform_vm_kernel/src/dma.rs | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/litebox_hal/src/virtio/pci.rs b/litebox_hal/src/virtio/pci.rs index 1f93b7c6be..065269db51 100644 --- a/litebox_hal/src/virtio/pci.rs +++ b/litebox_hal/src/virtio/pci.rs @@ -141,7 +141,9 @@ impl PciTransport { None => *slot.insert(unsafe { function.memory_bar(location.bar) }.ok_or(bad)?), }; let (offset, length) = (u64::from(location.offset), u64::from(location.length)); - if length < min_len as u64 || offset + length > bar.size { + // Fields are up to 4 bytes, naturally aligned; BARs are aligned + // to their size, so this aligns the addresses too. + if !offset.is_multiple_of(4) || length < min_len as u64 || offset + length > bar.size { return Err(bad); } let length = usize::try_from(length).map_err(|_| bad)?; diff --git a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs index 257a22725e..9a67d13282 100644 --- a/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs +++ b/litebox_platform_vm_kernel/src/arch/x86/interrupts.rs @@ -134,8 +134,9 @@ pub(crate) fn set_external_interrupt_handler(handler: &'static InterruptHandler) assert!(installed, "an interrupt handler is already set"); } -/// Runs with interrupts disabled, on whatever stack and GS the interrupted -/// context had (see `isr_external`). +/// Runs with interrupts disabled and the interrupted context's GS (see +/// `isr_external`), on the interrupted stack from kernel mode or on the +/// `TSS.RSP0` stack from user mode. #[unsafe(no_mangle)] extern "C" fn external_interrupt_handler_impl(vector: u64, rip: u64, cs: u64) { if let (Some(interrupts), Ok(vector)) = (INTERRUPTS.get(), u8::try_from(vector)) diff --git a/litebox_platform_vm_kernel/src/dma.rs b/litebox_platform_vm_kernel/src/dma.rs index 7363add5e2..982032db5b 100644 --- a/litebox_platform_vm_kernel/src/dma.rs +++ b/litebox_platform_vm_kernel/src/dma.rs @@ -36,8 +36,11 @@ unsafe impl Hal for VmKernel { .checked_add(u64::try_from(len).ok()?) .filter(|_| len != 0)?; let first = PhysFrame::::containing_address(PhysAddr::try_new(pa).ok()?); - let last = - PhysFrame::containing_address(PhysAddr::try_new(end).ok()?.align_up(Size4KiB::SIZE)); + // Not `PhysAddr::align_up`, which panics past the physical address + // width. + let last = PhysFrame::containing_address( + PhysAddr::try_new(end.checked_next_multiple_of(Size4KiB::SIZE)?).ok()?, + ); if self .ram_frame_ranges .iter() From eee1e65657b965d772adbf59226e011b7bdf3b08 Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Mon, 12 Oct 2026 16:21:15 +0000 Subject: [PATCH 7/7] nits --- litebox_broker_vm_kernel/src/stdio.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litebox_broker_vm_kernel/src/stdio.rs b/litebox_broker_vm_kernel/src/stdio.rs index ee1969a327..7290d721e0 100644 --- a/litebox_broker_vm_kernel/src/stdio.rs +++ b/litebox_broker_vm_kernel/src/stdio.rs @@ -54,7 +54,6 @@ impl ConsoleStdio { } } -/// Never a terminal. impl StdioProvider for ConsoleStdio { fn read(&self, output: &mut [u8]) -> Result { match self.console.read(output) { @@ -71,6 +70,7 @@ impl StdioProvider for ConsoleStdio { } } + /// No terminal control. fn is_terminal(&self, _stream: StdioStream) -> bool { false }