diff --git a/CHANGELOG.md b/CHANGELOG.md index c31b1e0..c4e9314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This file records each release. The format follows [Keep a Changelog](https://ke ## Unreleased +### Added + +- The eBPF verifier gives the result of kernel/bpf/verifier.c of Linux for all programs of bpf/test_verifier of the socket filter and cgroup program types. It prunes the paths with equal states at the jump targets and refuses a loop that does not end, it refines the bounds of a register after `BPF_JSET` and after a `BPF_JMP32` that compares as 64 bits, and it checks the alignment of the pointers and the size of each context field as Linux, with the errors and the log messages of Linux in the same order. `bpf(BPF_PROG_TEST_RUN)` runs a program on a packet or a context from user space. The kselftest boot test runs bpf/test_verifier, and tools/kselftest/bpf.md gives the result: 281 passed and 0 failed (#537). + ## [0.5.22] - 2026-11-12 ### Added diff --git a/core/rk-abi/src/errno.rs b/core/rk-abi/src/errno.rs index f9d3a1c..88f41e9 100644 --- a/core/rk-abi/src/errno.rs +++ b/core/rk-abi/src/errno.rs @@ -30,6 +30,9 @@ impl Errno { /// The buffer is too small, as `ETOOSMALL` of include/linux/errno.h. Linux gives it to user /// space from `ETHTOOL_GPERMADDR`, so user space sees this number. pub const ETOOSMALL: Self = Self(525); + /// The operation is not supported, as `ENOTSUPP` of include/linux/errno.h. `bpf(2)` gives it + /// to user space, for example from `BPF_PROG_TEST_RUN` of a program type without a test run. + pub const ENOTSUPP: Self = Self(524); /// Tells if the number is one of the restart numbers of the kernel. pub const fn is_restart(self) -> bool { diff --git a/core/rk-bpf/src/attr.rs b/core/rk-bpf/src/attr.rs index da9c738..9eed7d4 100644 --- a/core/rk-bpf/src/attr.rs +++ b/core/rk-bpf/src/attr.rs @@ -48,6 +48,8 @@ pub mod last_field { pub const BPF_PROG_ATTACH: usize = 32; /// `BPF_PROG_DETACH`: `expected_revision`. pub const BPF_PROG_DETACH: usize = 32; + /// `BPF_PROG_TEST_RUN`: `test.batch_size`. + pub const BPF_PROG_TEST_RUN: usize = 76; /// `BPF_PROG_QUERY`: `query.revision`. pub const BPF_PROG_QUERY: usize = 64; /// `BPF_LINK_CREATE`: `link_create.uprobe_multi.path_fd`. diff --git a/core/rk-bpf/src/core.rs b/core/rk-bpf/src/core.rs index 84f3d45..81fa887 100644 --- a/core/rk-bpf/src/core.rs +++ b/core/rk-bpf/src/core.rs @@ -668,7 +668,7 @@ impl Run<'_> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::insn::asm::*; use crate::map::MapEnv; @@ -694,7 +694,7 @@ mod tests { } } - fn prog(insns: &[BpfInsn], maps: Vec>) -> BpfProg { + pub(crate) fn prog(insns: &[BpfInsn], maps: Vec>) -> BpfProg { BpfProg { prog_type: 1, expected_attach_type: 0, diff --git a/core/rk-bpf/src/lib.rs b/core/rk-bpf/src/lib.rs index 4615da9..e453e1a 100644 --- a/core/rk-bpf/src/lib.rs +++ b/core/rk-bpf/src/lib.rs @@ -21,6 +21,7 @@ pub mod link; pub mod map; pub mod prog; pub mod syscall; +pub mod test_run; pub mod tnum; pub mod uapi; pub mod value; diff --git a/core/rk-bpf/src/prog.rs b/core/rk-bpf/src/prog.rs index 4801b6b..a71db55 100644 --- a/core/rk-bpf/src/prog.rs +++ b/core/rk-bpf/src/prog.rs @@ -240,6 +240,14 @@ pub struct SkBuff<'a> { pub remote_port: u32, /// The local port, in host byte order. pub local_port: u32, + /// `skb->tstamp`, the time of the packet in nanoseconds. + pub tstamp: u64, + /// `gso_segs` of the shared info of the packet. + pub gso_segs: u32, + /// `gso_size` of the shared info of the packet. + pub gso_size: u32, + /// The time stamp of the hardware, in nanoseconds. + pub hwtstamp: u64, } /// The offsets in `struct bpf_sock`, the context of `BPF_PROG_TYPE_CGROUP_SOCK`. @@ -509,8 +517,15 @@ pub mod sk_buff { pub const LOCAL_IP6: u32 = 116; pub const REMOTE_PORT: u32 = 132; pub const LOCAL_PORT: u32 = 136; + pub const DATA_META: u32 = 140; + pub const FLOW_KEYS: u32 = 144; + pub const TSTAMP: u32 = 152; + pub const WIRE_LEN: u32 = 160; pub const GSO_SEGS: u32 = 164; + pub const SK: u32 = 168; pub const GSO_SIZE: u32 = 176; + pub const TSTAMP_TYPE: u32 = 180; + pub const HWTSTAMP: u32 = 184; /// `sizeof(struct __sk_buff)`. pub const SIZE: u32 = 192; } @@ -525,6 +540,11 @@ impl ProgCtx for SkBuff<'_> { v[..b.len()].copy_from_slice(b); return Some(u64::from_le_bytes(v)); } + match off { + TSTAMP => return Some(self.tstamp), + HWTSTAMP => return Some(self.hwtstamp), + _ => {} + } // A narrow load reads the low bytes of the 32-bit field, as `bpf_ctx_narrow_access_ok`. let field = off & !3; let v: u32 = match field { @@ -549,8 +569,9 @@ impl ProgCtx for SkBuff<'_> { let at = (f - LOCAL_IP6) as usize; u32::from_ne_bytes(self.local_ip[at..at + 4].try_into().ok()?) } - QUEUE_MAPPING | VLAN_PRESENT | VLAN_TCI | VLAN_PROTO | TC_INDEX | NAPI_ID - | GSO_SEGS | GSO_SIZE => 0, + GSO_SEGS => self.gso_segs, + GSO_SIZE => self.gso_size, + QUEUE_MAPPING | VLAN_PRESENT | VLAN_TCI | VLAN_PROTO | TC_INDEX | NAPI_ID => 0, _ => return None, }; let shift = (off - field) * 8; @@ -563,6 +584,7 @@ impl ProgCtx for SkBuff<'_> { match off { MARK => self.mark = value as u32, PRIORITY => self.priority = value as u32, + TSTAMP => self.tstamp = value, _ if (CB..CB_END).contains(&off) => {} _ => return None, } diff --git a/core/rk-bpf/src/syscall.rs b/core/rk-bpf/src/syscall.rs index e35b167..06cae20 100644 --- a/core/rk-bpf/src/syscall.rs +++ b/core/rk-bpf/src/syscall.rs @@ -99,6 +99,8 @@ pub trait Env { fn boot_ns(&self) -> u64; /// The real user ID of the caller. fn uid(&self) -> u32; + /// What the helpers of a program see when the call runs it: the task of the caller. + fn helpers(&self) -> &dyn crate::helpers::HelperEnv; /// The cgroup of the cgroup2 directory `fd`, `cgroup_get_from_fd` of Linux. Another file /// gives `EBADF`. fn cgroup_get_from_fd(&self, fd: u32) -> Result; @@ -192,6 +194,7 @@ pub fn sys_bpf( BPF_PROG_ATTACH => bpf_prog_attach(env, &attr), BPF_PROG_DETACH => bpf_prog_detach(env, &attr), BPF_PROG_QUERY => bpf_prog_query(env, &attr, uattr, size), + BPF_PROG_TEST_RUN => bpf_prog_test_run(env, &attr, uattr), BPF_LINK_CREATE => link_create_cmd(env, &attr), BPF_LINK_UPDATE => link_update_cmd(env, &attr), BPF_LINK_DETACH => link_detach(env, &attr), @@ -400,6 +403,8 @@ fn bpf_prog_load(env: &dyn Env, attr: &BpfAttr, log_attr: &BpfLogAttr) -> Result expected_attach_type, bpf_capable: bpf_cap, allow_ptr_leaks: env.capable(CAP_PERFMON) || env.capable(CAP_SYS_ADMIN), + strict_alignment: prog_flags & BPF_F_STRICT_ALIGNMENT != 0 + && prog_flags & BPF_F_ANY_ALIGNMENT == 0, }; let map_by_fd = |fd: i32| -> Result> { let file = env.fdget(fd)?; @@ -1140,6 +1145,41 @@ fn bpf_obj_get_info_by_fd(env: &dyn Env, attr: &BpfAttr, uattr: u64) -> Result Result { + use crate::test_run::test; + if attr.check_attr(last_field::BPF_PROG_TEST_RUN) { + return Err(Errno::EINVAL); + } + let both = |size: usize, ptr: usize| (attr.u32(size) == 0) == (attr.u64(ptr) == 0); + if !both(test::CTX_SIZE_IN, test::CTX_IN) || !both(test::CTX_SIZE_OUT, test::CTX_OUT) { + return Err(Errno::EINVAL); + } + let prog = bpf_prog_get(env, attr.u32(test::PROG_FD))?; + crate::test_run::bpf_prog_test_run(&EnvTestRun(env), &prog, attr, uattr) +} + +/// The caller of a test run. +struct EnvTestRun<'a>(&'a dyn Env); + +impl crate::test_run::TestRunEnv for EnvTestRun<'_> { + fn copy_from_user(&self, addr: u64, buf: &mut [u8]) -> Result<()> { + self.0.copy_from_user(addr, buf) + } + + fn copy_to_user(&self, addr: u64, buf: &[u8]) -> Result<()> { + self.0.copy_to_user(addr, buf) + } + + fn helpers(&self) -> &dyn crate::helpers::HelperEnv { + self.0.helpers() + } + + fn now_ns(&self) -> u64 { + self.0.boot_ns() + } +} + /// The program of a descriptor, `bpf_prog_get` of Linux. fn bpf_prog_get(env: &dyn Env, fd: u32) -> Result> { bpf_prog_of(&*env.fdget(fd as i32)?).ok_or(Errno::EINVAL) @@ -1592,6 +1632,10 @@ mod tests { 1000 } + fn helpers(&self) -> &dyn crate::helpers::HelperEnv { + &crate::core::tests::NoEnv + } + // The descriptors from 5000 are the cgroups 5000 and up, each one the child of the // one before. fn cgroup_get_from_fd(&self, fd: u32) -> Result { diff --git a/core/rk-bpf/src/test_run.rs b/core/rk-bpf/src/test_run.rs new file mode 100644 index 0000000..e6bc3e7 --- /dev/null +++ b/core/rk-bpf/src/test_run.rs @@ -0,0 +1,528 @@ +//! `BPF_PROG_TEST_RUN`, net/bpf/test_run.c of Linux. +//! +//! The command runs a program once or more on a packet of the caller, and gives the result of +//! the last run. The socket filters and the `BPF_PROG_TYPE_CGROUP_SKB` programs run on a packet +//! that comes in on the loopback device, as `bpf_prog_test_run_skb` of Linux. The other program +//! types of rukernel have no test run, and give `ENOTSUPP`, as on Linux. + +use crate::attr::BpfAttr; +use crate::helpers::HelperEnv; +use crate::prog::prog_type::{BPF_PROG_TYPE_CGROUP_SKB, BPF_PROG_TYPE_SOCKET_FILTER}; +use crate::prog::{BpfProg, SkBuff, sk_buff}; +use alloc::vec; +use alloc::vec::Vec; +use rk_abi::Errno; + +type Result = core::result::Result; + +/// The fields of `BPF_PROG_TEST_RUN`, `test`. +pub mod test { + /// `prog_fd`. + pub const PROG_FD: usize = 0; + /// `retval`. + pub const RETVAL: usize = 4; + /// `data_size_in`. + pub const DATA_SIZE_IN: usize = 8; + /// `data_size_out`. + pub const DATA_SIZE_OUT: usize = 12; + /// `data_in`. + pub const DATA_IN: usize = 16; + /// `data_out`. + pub const DATA_OUT: usize = 24; + /// `repeat`. + pub const REPEAT: usize = 32; + /// `duration`. + pub const DURATION: usize = 36; + /// `ctx_size_in`. + pub const CTX_SIZE_IN: usize = 40; + /// `ctx_size_out`. + pub const CTX_SIZE_OUT: usize = 44; + /// `ctx_in`. + pub const CTX_IN: usize = 48; + /// `ctx_out`. + pub const CTX_OUT: usize = 56; + /// `flags`. + pub const FLAGS: usize = 64; + /// `cpu`. + pub const CPU: usize = 68; + /// `batch_size`. + pub const BATCH_SIZE: usize = 72; +} + +/// The error of a program type without a test run, `ENOTSUPP` of include/linux/errno.h. +pub const ENOTSUPP: Errno = Errno::ENOTSUPP; + +/// The size of the Ethernet header, `ETH_HLEN`. +const ETH_HLEN: usize = 14; +/// The largest packet: `PAGE_SIZE` less the head room and the tail room of the buffer, as +/// `bpf_test_init` of Linux on x86_64 and arm64. +const MAX_DATA: usize = 4096 - (64 + 2) - 320; +/// `BPF_F_TEST_SKB_CHECKSUM_COMPLETE`. +const BPF_F_TEST_SKB_CHECKSUM_COMPLETE: u32 = 1 << 2; +/// The index of the loopback device, which gets the packet. +const LOOPBACK_IFINDEX: u32 = 1; +/// `GSO_LEGACY_MAX_SIZE`. +const GSO_LEGACY_MAX_SIZE: u32 = 65536; +/// `GSO_MAX_SEGS`. +const GSO_MAX_SEGS: u32 = 65535; + +/// The offsets in `struct __sk_buff`, as `usize`. +mod skb { + use crate::prog::sk_buff as o; + pub(super) const TC_INDEX: usize = o::TC_INDEX as usize; + pub(super) const TSTAMP: usize = o::TSTAMP as usize; + pub(super) const WIRE_LEN: usize = o::WIRE_LEN as usize; + pub(super) const GSO_SEGS: usize = o::GSO_SEGS as usize; + pub(super) const SK: usize = o::SK as usize; + pub(super) const GSO_SIZE: usize = o::GSO_SIZE as usize; + pub(super) const TSTAMP_TYPE: usize = o::TSTAMP_TYPE as usize; + pub(super) const HWTSTAMP: usize = o::HWTSTAMP as usize; +} + +/// What the test run needs of the caller. +pub trait TestRunEnv { + /// Copies the user memory at `addr` to `buf`. A bad address gives `EFAULT`. + fn copy_from_user(&self, addr: u64, buf: &mut [u8]) -> Result<()>; + /// Copies `buf` to the user memory at `addr`. A bad address gives `EFAULT`. + fn copy_to_user(&self, addr: u64, buf: &[u8]) -> Result<()>; + /// What the helpers see: the task of the caller. + fn helpers(&self) -> &dyn HelperEnv; + /// The nanoseconds after the boot. + fn now_ns(&self) -> u64; +} + +/// Runs the test run of the program type, as the `test_run` operation of Linux. +pub fn bpf_prog_test_run( + env: &dyn TestRunEnv, + prog: &BpfProg, + attr: &BpfAttr, + uattr: u64, +) -> Result { + match prog.prog_type { + BPF_PROG_TYPE_SOCKET_FILTER | BPF_PROG_TYPE_CGROUP_SKB => { + bpf_prog_test_run_skb(env, prog, attr, uattr) + } + _ => Err(ENOTSUPP), + } +} + +/// Copies the packet of the caller, as `bpf_test_init` of Linux. +fn bpf_test_init(env: &dyn TestRunEnv, attr: &BpfAttr) -> Result> { + let size = attr.u32(test::DATA_SIZE_IN) as usize; + if !(ETH_HLEN..=MAX_DATA).contains(&size) { + return Err(Errno::EINVAL); + } + let mut data = vec![0; size]; + env.copy_from_user(attr.u64(test::DATA_IN), &mut data)?; + Ok(data) +} + +/// Copies the context of the caller, as `bpf_ctx_init` of Linux. No context gives `None`. +fn bpf_ctx_init(env: &dyn TestRunEnv, attr: &BpfAttr, max_size: usize) -> Result>> { + let data_in = attr.u64(test::CTX_IN); + if data_in == 0 && attr.u64(test::CTX_OUT) == 0 { + return Ok(None); + } + let mut data = vec![0; max_size]; + if data_in != 0 { + let size = attr.u32(test::CTX_SIZE_IN) as usize; + let n = crate::attr::bpf_check_uarg_tail_zero(max_size, size, |rest| { + env.copy_from_user(data_in + max_size as u64, rest) + })?; + env.copy_from_user(data_in, &mut data[..n])?; + } + Ok(Some(data)) +} + +/// Tells if the bytes of `ctx` from `from` to `to` are zero, `range_is_zero` of Linux. +fn range_is_zero(ctx: &[u8], from: usize, to: usize) -> bool { + ctx[from..to].iter().all(|&b| b == 0) +} + +fn get_u32(ctx: &[u8], off: usize) -> u32 { + u32::from_ne_bytes([ctx[off], ctx[off + 1], ctx[off + 2], ctx[off + 3]]) +} + +fn get_u64(ctx: &[u8], off: usize) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(&ctx[off..off + 8]); + u64::from_ne_bytes(b) +} + +fn put_u32(ctx: &mut [u8], off: usize, v: u32) { + ctx[off..off + 4].copy_from_slice(&v.to_ne_bytes()); +} + +/// The state of the packet that the context of the caller sets, and that the caller gets back. +#[derive(Default)] +struct SkbState { + mark: u32, + priority: u32, + skb_iif: u32, + cb: [u8; 20], + tstamp: u64, + pkt_len: u32, + gso_segs: u32, + gso_size: u32, + hwtstamp: u64, +} + +/// Checks the context of the caller and takes its fields, `convert___skb_to_skb` of Linux. Each +/// field that the caller cannot set must be zero. +fn skb_from_user(ctx: &[u8], len: u32, st: &mut SkbState) -> Result<()> { + use sk_buff::{CB, CB_END, IFINDEX, MARK, PRIORITY}; + let (mark, priority, ifindex, cb) = (MARK as usize, PRIORITY as usize, IFINDEX as usize, CB); + let zero = [ + (0, mark), + (mark + 4, priority), + (ifindex + 4, cb as usize), + (CB_END as usize, skb::TSTAMP), + (skb::GSO_SEGS + 4, skb::GSO_SIZE), + (skb::GSO_SIZE + 4, skb::HWTSTAMP), + ]; + if zero.iter().any(|&(from, to)| !range_is_zero(ctx, from, to)) { + return Err(Errno::EINVAL); + } + const _: () = assert!( + skb::TC_INDEX == sk_buff::IFINDEX as usize + 4 + && skb::SK < skb::GSO_SIZE + && skb::TSTAMP_TYPE == skb::GSO_SIZE + 4 + ); + st.mark = get_u32(ctx, mark); + st.priority = get_u32(ctx, priority); + st.skb_iif = get_u32(ctx, sk_buff::INGRESS_IFINDEX as usize); + st.tstamp = get_u64(ctx, skb::TSTAMP); + st.cb.copy_from_slice(&ctx[cb as usize..CB_END as usize]); + let wire_len = get_u32(ctx, skb::WIRE_LEN); + st.pkt_len = if wire_len == 0 { + len + } else { + if wire_len < len || wire_len > GSO_LEGACY_MAX_SIZE { + return Err(Errno::EINVAL); + } + wire_len + }; + st.gso_segs = get_u32(ctx, skb::GSO_SEGS); + if st.gso_segs > GSO_MAX_SEGS { + return Err(Errno::EINVAL); + } + st.gso_size = get_u32(ctx, skb::GSO_SIZE); + st.hwtstamp = get_u64(ctx, skb::HWTSTAMP); + Ok(()) +} + +/// Gives the caller the fields of the packet after the run, `convert_skb_to___skb` of Linux. +fn skb_to_user(st: &SkbState, ctx: &mut [u8]) { + put_u32(ctx, sk_buff::MARK as usize, st.mark); + put_u32(ctx, sk_buff::PRIORITY as usize, st.priority); + put_u32(ctx, sk_buff::INGRESS_IFINDEX as usize, st.skb_iif); + put_u32(ctx, sk_buff::IFINDEX as usize, LOOPBACK_IFINDEX); + ctx[skb::TSTAMP..skb::TSTAMP + 8].copy_from_slice(&st.tstamp.to_ne_bytes()); + ctx[sk_buff::CB as usize..sk_buff::CB_END as usize].copy_from_slice(&st.cb); + put_u32(ctx, skb::WIRE_LEN, st.pkt_len); + put_u32(ctx, skb::GSO_SEGS, st.gso_segs); + ctx[skb::HWTSTAMP..skb::HWTSTAMP + 8].copy_from_slice(&st.hwtstamp.to_ne_bytes()); +} + +/// The protocol and the packet type of an Ethernet frame on the loopback device, as +/// `eth_type_trans` of Linux. The protocol is in network byte order. +fn eth_type_trans(frame: &[u8]) -> (u16, u32) { + let dest = &frame[..6]; + // The loopback device has the address 00:00:00:00:00:00. + let pkt_type = if dest.iter().all(|&b| b == 0) { + 0 // PACKET_HOST + } else if dest[0] & 1 != 0 { + if dest.iter().all(|&b| b == 0xff) { 1 } else { 2 } // PACKET_BROADCAST, PACKET_MULTICAST + } else { + 3 // PACKET_OTHERHOST + }; + let proto = u16::from_be_bytes([frame[12], frame[13]]); + let proto = if proto >= 0x600 { + proto + } else if frame.get(14..16) == Some(&[0xff, 0xff]) { + 1 // ETH_P_802_3 + } else { + 4 // ETH_P_802_2 + }; + (proto.to_be(), pkt_type) +} + +/// Gives the caller the packet and the result, `bpf_test_finish` of Linux. A buffer that is too +/// small gets the start of the packet, and the call gives `ENOSPC`. +fn bpf_test_finish( + env: &dyn TestRunEnv, + attr: &BpfAttr, + uattr: u64, + data: &[u8], + retval: u32, + duration: u32, +) -> Result<()> { + let size = data.len() as u32; + let size_out = attr.u32(test::DATA_SIZE_OUT); + let mut copy = size; + let mut err = Ok(()); + if size_out != 0 && copy > size_out { + copy = size_out; + err = Err(Errno::ENOSPC); + } + let data_out = attr.u64(test::DATA_OUT); + if data_out != 0 { + env.copy_to_user(data_out, &data[..copy as usize])?; + } + env.copy_to_user(uattr + test::DATA_SIZE_OUT as u64, &size.to_ne_bytes())?; + env.copy_to_user(uattr + test::RETVAL as u64, &retval.to_ne_bytes())?; + env.copy_to_user(uattr + test::DURATION as u64, &duration.to_ne_bytes())?; + err +} + +/// Gives the caller the context after the run, `bpf_ctx_finish` of Linux. +fn bpf_ctx_finish(env: &dyn TestRunEnv, attr: &BpfAttr, uattr: u64, ctx: &[u8]) -> Result<()> { + let data_out = attr.u64(test::CTX_OUT); + if data_out == 0 { + return Ok(()); + } + let size = ctx.len() as u32; + let size_out = attr.u32(test::CTX_SIZE_OUT); + let (copy, err) = if size > size_out { (size_out, Err(Errno::ENOSPC)) } else { (size, Ok(())) }; + env.copy_to_user(data_out, &ctx[..copy as usize])?; + env.copy_to_user(uattr + test::CTX_SIZE_OUT as u64, &size.to_ne_bytes())?; + err +} + +/// Runs the program `repeat` times, at least once, as `bpf_test_run` of Linux. Gives the result +/// of the last run and the mean time of a run in nanoseconds. +fn bpf_test_run( + env: &dyn TestRunEnv, + prog: &BpfProg, + skb: &mut SkBuff<'_>, + repeat: u32, +) -> (u32, u32) { + let repeat = repeat.max(1); + let start = env.now_ns(); + let mut retval = 0; + for _ in 0..repeat { + retval = prog.run(skb, env.helpers()) as u32; + } + let ns = env.now_ns().saturating_sub(start) / u64::from(repeat); + (retval, ns.min(u64::from(u32::MAX)) as u32) +} + +/// The test run of the programs of a packet, `bpf_prog_test_run_skb` of Linux. +fn bpf_prog_test_run_skb( + env: &dyn TestRunEnv, + prog: &BpfProg, + attr: &BpfAttr, + uattr: u64, +) -> Result { + if attr.u32(test::FLAGS) & !BPF_F_TEST_SKB_CHECKSUM_COMPLETE != 0 + || attr.u32(test::CPU) != 0 + || attr.u32(test::BATCH_SIZE) != 0 + { + return Err(Errno::EINVAL); + } + let mut data = bpf_test_init(env, attr)?; + let mut ctx = bpf_ctx_init(env, attr, sk_buff::SIZE as usize)?; + if let Some(c) = &ctx { + // rukernel gives the test run only the loopback device. + if get_u32(c, sk_buff::IFINDEX as usize) > LOOPBACK_IFINDEX { + return Err(Errno::ENODEV); + } + } + let (protocol, pkt_type) = eth_type_trans(&data); + let net = &data[ETH_HLEN..]; + // The socket of the packet has the addresses of the packet, as the dummy socket of Linux. + let (mut family, mut local_ip, mut remote_ip) = (0, [0u8; 16], [0u8; 16]); + match u16::from_be(protocol) { + 0x0800 => { + family = 2; + if net.len() >= 20 { + local_ip[..4].copy_from_slice(&net[12..16]); + remote_ip[..4].copy_from_slice(&net[16..20]); + } + } + 0x86dd => { + family = 10; + if net.len() >= 40 { + local_ip.copy_from_slice(&net[8..24]); + remote_ip.copy_from_slice(&net[24..40]); + } + } + _ => {} + } + let mut st = SkbState::default(); + if let Some(c) = &ctx { + skb_from_user(c, net.len() as u32, &mut st)?; + } + let mut skb = SkBuff { + data: net, + protocol, + pkt_type, + mark: st.mark, + priority: st.priority, + ingress_ifindex: st.skb_iif, + ifindex: LOOPBACK_IFINDEX, + cb: st.cb, + family, + remote_ip, + local_ip, + tstamp: st.tstamp, + gso_segs: st.gso_segs, + gso_size: st.gso_size, + hwtstamp: st.hwtstamp, + ..SkBuff::default() + }; + let (retval, duration) = bpf_test_run(env, prog, &mut skb, attr.u32(test::REPEAT)); + st.mark = skb.mark; + st.priority = skb.priority; + st.cb = skb.cb; + st.tstamp = skb.tstamp; + // The Ethernet header goes back on the packet with zero bytes. + data[..ETH_HLEN].fill(0); + if let Some(c) = &mut ctx { + skb_to_user(&st, c); + } + bpf_test_finish(env, attr, uattr, &data, retval, duration)?; + if let Some(c) = &ctx { + bpf_ctx_finish(env, attr, uattr, c)?; + } + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::tests::NoEnv; + use crate::insn::asm::*; + use crate::insn::*; + use alloc::sync::Arc; + use core::cell::RefCell; + + /// A caller with a flat memory from address 0x1000. + struct Mem(RefCell>); + + const BASE: u64 = 0x1000; + const UATTR: u64 = BASE; + const DATA: u64 = BASE + 0x200; + const OUT: u64 = BASE + 0x400; + const CTX: u64 = BASE + 0x800; + + impl TestRunEnv for Mem { + fn copy_from_user(&self, addr: u64, buf: &mut [u8]) -> Result<()> { + let at = (addr - BASE) as usize; + let m = self.0.borrow(); + buf.copy_from_slice(m.get(at..at + buf.len()).ok_or(Errno::EFAULT)?); + Ok(()) + } + fn copy_to_user(&self, addr: u64, buf: &[u8]) -> Result<()> { + let at = (addr - BASE) as usize; + let mut m = self.0.borrow_mut(); + m.get_mut(at..at + buf.len()).ok_or(Errno::EFAULT)?.copy_from_slice(buf); + Ok(()) + } + fn helpers(&self) -> &dyn HelperEnv { + &NoEnv + } + fn now_ns(&self) -> u64 { + 0 + } + } + + fn prog(prog_type: u32, insns: &[BpfInsn]) -> Arc { + let mut p = crate::core::tests::prog(insns, Vec::new()); + p.prog_type = prog_type; + Arc::new(p) + } + + fn get(m: &Mem, addr: u64, len: usize) -> Vec { + let mut v = vec![0; len]; + m.copy_from_user(addr, &mut v).unwrap(); + v + } + + fn attr(size_in: u32, size_out: u32) -> BpfAttr { + let mut a = BpfAttr::zeroed(); + a.set_u32(test::DATA_SIZE_IN, size_in); + a.set_u64(test::DATA_IN, DATA); + a.set_u32(test::DATA_SIZE_OUT, size_out); + a.set_u64(test::DATA_OUT, OUT); + a + } + + #[test] + fn a_socket_filter_runs_on_the_packet_after_the_ethernet_header() { + let m = Mem(RefCell::new(vec![0; 0x1000])); + let mut frame = vec![0u8; 64]; + frame[..6].copy_from_slice(&[0xff; 6]); + frame[12..14].copy_from_slice(&[0x08, 0x00]); + frame[14] = 0x45; + m.copy_to_user(DATA, &frame).unwrap(); + // r0 = skb->len + skb->pkt_type * 1000 + the first byte of the network header. + let p = prog( + BPF_PROG_TYPE_SOCKET_FILTER, + &[ + mov64_reg(6, 1), + ldx_mem(BPF_W, 0, 6, sk_buff::LEN as i16), + ldx_mem(BPF_W, 2, 6, sk_buff::PKT_TYPE as i16), + alu64_imm(BPF_MUL, 2, 1000), + alu64_reg(BPF_ADD, 0, 2), + mov64_reg(7, 0), + ld_abs(BPF_B, 0), + alu64_reg(BPF_ADD, 0, 7), + exit(), + ], + ); + bpf_prog_test_run(&m, &p, &attr(64, 128), UATTR).unwrap(); + let out = get(&m, UATTR, 40); + let u = |o: usize| u32::from_ne_bytes(out[o..o + 4].try_into().unwrap()); + assert_eq!(u(test::RETVAL), 50 + 1000 + 0x45); + assert_eq!(u(test::DATA_SIZE_OUT), 64); + let data = get(&m, OUT, 64); + assert_eq!(&data[..14], &[0; 14]); + assert_eq!(&data[14..], &frame[14..]); + } + + #[test] + fn the_errors_of_a_test_run() { + let m = Mem(RefCell::new(vec![0; 0x1000])); + let p = prog(BPF_PROG_TYPE_SOCKET_FILTER, &[mov64_imm(0, 1), exit()]); + assert_eq!(bpf_prog_test_run(&m, &p, &attr(13, 0), UATTR), Err(Errno::EINVAL)); + let mut a = attr(14, 0); + a.set_u32(test::CPU, 1); + assert_eq!(bpf_prog_test_run(&m, &p, &a, UATTR), Err(Errno::EINVAL)); + // A small buffer gets the start of the packet. + assert_eq!(bpf_prog_test_run(&m, &p, &attr(20, 16), UATTR), Err(Errno::ENOSPC)); + let dev = prog(crate::prog::prog_type::BPF_PROG_TYPE_CGROUP_DEVICE, &[exit()]); + assert_eq!(bpf_prog_test_run(&m, &dev, &attr(14, 0), UATTR), Err(ENOTSUPP)); + } + + #[test] + fn the_context_of_a_test_run() { + let m = Mem(RefCell::new(vec![0; 0x1000])); + let p = prog( + BPF_PROG_TYPE_SOCKET_FILTER, + &[ + ldx_mem(BPF_W, 0, 1, sk_buff::MARK as i16), + mov64_imm(2, 7), + stx_mem(BPF_W, 1, 2, sk_buff::CB as i16), + exit(), + ], + ); + let mut ctx = vec![0u8; sk_buff::SIZE as usize]; + put_u32(&mut ctx, sk_buff::MARK as usize, 42); + m.copy_to_user(CTX, &ctx).unwrap(); + let mut a = attr(14, 0); + a.set_u32(test::CTX_SIZE_IN, sk_buff::SIZE); + a.set_u64(test::CTX_IN, CTX); + a.set_u32(test::CTX_SIZE_OUT, sk_buff::SIZE); + a.set_u64(test::CTX_OUT, CTX); + bpf_prog_test_run(&m, &p, &a, UATTR).unwrap(); + assert_eq!(u32::from_ne_bytes(get(&m, UATTR + 4, 4).try_into().unwrap()), 42); + let out = get(&m, CTX, sk_buff::SIZE as usize); + assert_eq!(get_u32(&out, sk_buff::CB as usize), 7); + assert_eq!(get_u32(&out, sk_buff::IFINDEX as usize), 1); + // A field that the caller cannot set gives EINVAL. + put_u32(&mut ctx, sk_buff::LEN as usize, 1); + m.copy_to_user(CTX, &ctx).unwrap(); + assert_eq!(bpf_prog_test_run(&m, &p, &a, UATTR), Err(Errno::EINVAL)); + } +} diff --git a/core/rk-bpf/src/uapi.rs b/core/rk-bpf/src/uapi.rs index 18dc5eb..5d3af26 100644 --- a/core/rk-bpf/src/uapi.rs +++ b/core/rk-bpf/src/uapi.rs @@ -22,6 +22,8 @@ pub mod cmd { pub const BPF_PROG_ATTACH: u32 = 8; /// Detaches a program from a hook. pub const BPF_PROG_DETACH: u32 = 9; + /// Runs a program on a packet of the caller. + pub const BPF_PROG_TEST_RUN: u32 = 10; /// Gives the program ID after `start_id`. pub const BPF_PROG_GET_NEXT_ID: u32 = 11; /// Gives the map ID after `start_id`. diff --git a/core/rk-bpf/src/verifier.rs b/core/rk-bpf/src/verifier.rs index e3c2ec8..e7b0e71 100644 --- a/core/rk-bpf/src/verifier.rs +++ b/core/rk-bpf/src/verifier.rs @@ -5,15 +5,17 @@ //! stops. This first verifier is smaller than the verifier of Linux, and it refuses what it //! cannot prove: //! -//! - `check_cfg` refuses back edges, so there are no loops, and each instruction must be -//! reachable. +//! - `check_cfg` refuses back edges for a loader without `CAP_BPF`, and each instruction must be +//! reachable. With `CAP_BPF`, `do_check` follows each round of a loop, so a loop must end +//! within `BPF_COMPLEXITY_LIMIT_INSNS`. //! - `do_check` follows each path of the program with the state of the registers and of the //! stack. It does not prune states, so the number of paths is limited by //! `BPF_COMPLEXITY_LIMIT_INSNS`. //! - A scalar has a tnum and the unsigned and signed bounds. A pointer is to the context, to the //! stack, to a map, or to a map value that can be NULL until the program checks it. //! - A call of a function of the program gets a new frame, and the verifier checks the function -//! again at each call, as Linux does for a static function. Recursion is a back edge. +//! again at each call, as Linux does for a static function. Recursion stops at the limit of +//! the frames. //! - `BPF_LD_[ABS|IND]` and tail calls are allowed only in the main program, because rukernel //! has no BTF. Kernel functions and callbacks are refused. //! @@ -30,6 +32,7 @@ use crate::tnum::{TNUM_UNKNOWN, Tnum}; use crate::uapi::attach_type::*; use crate::uapi::flags::{BPF_F_RDONLY_PROG, BPF_F_WRONLY_PROG}; use crate::uapi::map_type::{BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PROG_ARRAY}; +use alloc::boxed::Box; use alloc::string::String; use alloc::sync::Arc; use alloc::vec; @@ -52,6 +55,11 @@ pub const BPF_LOG_MASK: u32 = BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2 | BPF_LOG_STATS | /// The most states that wait to be checked, `BPF_COMPLEXITY_LIMIT_JMP_SEQ` of Linux. const BPF_COMPLEXITY_LIMIT_JMP_SEQ: usize = 8192; +/// The most states that `do_check` keeps for the pruning. A state is about 5 KiB. +const MAX_KEPT_STATES: usize = 1024; +/// The most states that `do_check` keeps for one instruction. The states of a loop are all +/// different, and each round compares the state with each kept state. +const MAX_KEPT_STATES_PER_INSN: usize = 16; /// The largest offset of a pointer, `BPF_MAX_VAR_OFF` of Linux. const BPF_MAX_VAR_OFF: i64 = 1 << 29; @@ -241,12 +249,12 @@ impl StackSlot { } /// The state of a path, `struct bpf_verifier_state` of Linux. `regs` and `stack` are of the -/// frame that runs now. -#[derive(Clone, Debug)] +/// frame that runs now. The registers and the stack are on the heap, because the kernel stack +/// is small. A debug build keeps many copies of a state on the stack when the state is large. +#[derive(Clone, Debug, PartialEq, Eq)] struct State { - regs: [Reg; MAX_BPF_REG], - /// The slots of the stack. They are on the heap, because a state is large and the kernel - /// stack is small. + regs: Box<[Reg; MAX_BPF_REG]>, + /// The slots of the stack. stack: Vec, /// The function of the program that runs now. subprog: usize, @@ -255,9 +263,9 @@ struct State { } /// The state of a caller, `struct bpf_func_state` of Linux. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] struct Frame { - regs: [Reg; MAX_BPF_REG], + regs: Box<[Reg; MAX_BPF_REG]>, stack: Vec, subprog: usize, /// The instruction of the call. @@ -301,10 +309,24 @@ const BPF_MAX_SUBPROGS: usize = 256; const MAX_USED_MAPS: usize = 64; /// `ENOTSUPP` of include/linux/errno.h. Linux gives this number of the kernel to user space /// when the kernel cannot call a kernel function. -const ENOTSUPP: Errno = match Errno::new(524) { - Some(e) => e, - None => Errno::EINVAL, -}; +const ENOTSUPP: Errno = Errno::ENOTSUPP; + +/// Shows a tnum as `tnum_strn` of Linux, which prints with `%#llx`: 0 is `0`, and other numbers +/// have `0x`. +struct TnumStr(Tnum); + +impl core::fmt::Display for TnumStr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let hex = |f: &mut core::fmt::Formatter<'_>, v: u64| { + if v == 0 { f.write_str("0") } else { write!(f, "{v:#x}") } + }; + f.write_str("(")?; + hex(f, self.0.value)?; + f.write_str("; ")?; + hex(f, self.0.mask)?; + f.write_str(")") + } +} /// The verifier, `struct bpf_verifier_env` of Linux. struct Env<'a> { @@ -314,12 +336,18 @@ struct Env<'a> { expected_attach_type: u32, bpf_capable: bool, allow_ptr_leaks: bool, + /// The loader gave `BPF_F_STRICT_ALIGNMENT`, `strict_alignment` of Linux. + strict_alignment: bool, + /// The access of an atomic instruction must be aligned, `strict_alignment_once` of Linux. + strict_once: bool, log: &'a mut BpfVerifierLog, enforce_expected_attach_type: bool, id_gen: u32, insn_processed: u32, /// The functions of the program, ordered by their first instruction. subprogs: Vec, + /// The type of the pointer of each load or store, `insn_aux_data[].ptr_type` of Linux. + ptr_types: Vec, } /// What the program does with a memory access. @@ -351,6 +379,9 @@ pub struct VerifierOpts { pub bpf_capable: bool, /// The loader has `CAP_PERFMON`, so a program can show addresses, `allow_ptr_leaks` of Linux. pub allow_ptr_leaks: bool, + /// Each access of memory must be aligned to its size, as `BPF_F_STRICT_ALIGNMENT`. x86_64 + /// and arm64 have efficient unaligned access, so this is false without the flag. + pub strict_alignment: bool, } /// Tells if the verifier knows the program type, as `find_prog_type` of Linux. @@ -383,11 +414,14 @@ pub fn bpf_check( expected_attach_type: opts.expected_attach_type, bpf_capable: opts.bpf_capable, allow_ptr_leaks: opts.allow_ptr_leaks, + strict_alignment: opts.strict_alignment, + strict_once: false, log: &mut *log, enforce_expected_attach_type: false, id_gen: 0, insn_processed: 0, subprogs: Vec::new(), + ptr_types: Vec::new(), }; let r = env .add_subprogs() @@ -410,11 +444,14 @@ pub fn bpf_check( expected_attach_type: opts.expected_attach_type, bpf_capable: opts.bpf_capable, allow_ptr_leaks: opts.allow_ptr_leaks, + strict_alignment: opts.strict_alignment, + strict_once: false, log, enforce_expected_attach_type: false, id_gen: 0, insn_processed: 0, subprogs, + ptr_types: vec![RegType::NotInit; insns.len()], }; let r = env .check_cfg() @@ -851,7 +888,7 @@ impl Env<'_> { } let w = w as usize; if half[w] { - verbose!(self, "jump into the middle of ldimm64 insn {}\n", w); + verbose!(self, "jump into the middle of ldimm64 insn {}\n", w - 1); return Err(Errno::EINVAL); } match state[w] { @@ -859,6 +896,10 @@ impl Env<'_> { state[w] = DISCOVERED; stack.push((w, 0)); } + // A loader with `CAP_BPF` can have loops. `do_check` follows each round of a + // loop, and refuses a loop that does not end within + // `BPF_COMPLEXITY_LIMIT_INSNS`. + DISCOVERED if self.bpf_capable => {} DISCOVERED => { verbose!(self, "back-edge from insn {} to {}\n", t, w); return Err(Errno::EINVAL); @@ -881,14 +922,40 @@ impl Env<'_> { /// Follows each path of the program, as `do_check` of Linux. fn do_check(&mut self) -> Result<()> { - let mut regs = [Reg::NOT_INIT; MAX_BPF_REG]; + let mut regs = Box::new([Reg::NOT_INIT; MAX_BPF_REG]); regs[1] = Reg::ptr(RegType::PtrToCtx); regs[BPF_REG_FP as usize] = Reg::ptr(RegType::PtrToStack); let st = State { regs, stack: vec![StackSlot::EMPTY; SLOTS], subprog: 0, callers: Vec::new() }; - let mut pending: Vec<(usize, State)> = vec![(0, st)]; - while let Some((mut pc, mut st)) = pending.pop() { + // The pruning of `is_state_visited` of Linux, with states that are equal. A state that + // is equal to a state of the same instruction gives the same result, so the path stops. + // When the earlier state is on the current path, the path is a loop that does not end. + // A path that waits in `pending` remembers the length of `path` when it starts, because + // the states after that length are of a branch that is complete when it starts. + let prune = self.prune_points(); + let mut kept: Vec> = vec![Vec::new(); self.insns.len()]; + let mut nkept = 0; + let mut path: Vec<(usize, usize)> = Vec::new(); + let mut pending: Vec<(usize, State, usize)> = vec![(0, st, 0)]; + while let Some((mut pc, mut st, depth)) = pending.pop() { + for (p, i) in path.drain(depth..) { + kept[p][i].1 = false; + } loop { + if prune[pc] { + if let Some(&(_, on_path)) = kept[pc].iter().find(|(s, _)| *s == st) { + if on_path { + verbose!(self, "infinite loop detected at insn {}\n", pc); + return Err(Errno::EINVAL); + } + break; + } + if nkept < MAX_KEPT_STATES && kept[pc].len() < MAX_KEPT_STATES_PER_INSN { + kept[pc].push((st.clone(), true)); + path.push((pc, kept[pc].len() - 1)); + nkept += 1; + } + } self.insn_processed += 1; if self.insn_processed > BPF_COMPLEXITY_LIMIT_INSNS { verbose!( @@ -912,7 +979,7 @@ impl Env<'_> { ); return Err(Errno::E2BIG); } - pending.push((other, *other_state)); + pending.push((other, *other_state, path.len())); pc = next; } Next::Exit => break, @@ -922,8 +989,30 @@ impl Env<'_> { Ok(()) } + /// The instructions where `do_check` compares the states, as `mark_prune_point` of Linux: + /// the targets of the jumps and the instructions after the conditional jumps. + fn prune_points(&self) -> Vec { + let n = self.insns.len(); + let mut prune = vec![false; n]; + for i in 0..n { + let class = bpf_class(self.insns[i].code); + if (class != BPF_JMP && class != BPF_JMP32) + || matches!(bpf_op(self.insns[i].code), BPF_EXIT | BPF_CALL) + { + continue; + } + for w in successors(self.insns, i).into_iter().flatten() { + if (0..n as i64).contains(&w) { + prune[w as usize] = true; + } + } + } + prune + } + /// Reads a register, as `check_reg_arg` with `SRC_OP` of Linux. fn read_reg(&mut self, st: &State, regno: u8) -> Result { + self.check_regno(regno)?; let r = st.regs[regno as usize]; if r.ty == RegType::NotInit { verbose!(self, "R{} !read_ok\n", regno); @@ -934,6 +1023,7 @@ impl Env<'_> { /// Checks that a register can be written, as `check_reg_arg` with `DST_OP` of Linux. fn check_dst(&mut self, regno: u8) -> Result<()> { + self.check_regno(regno)?; if regno == BPF_REG_FP { verbose!(self, "frame pointer is read only\n"); return Err(Errno::EACCES); @@ -941,13 +1031,25 @@ impl Env<'_> { Ok(()) } + /// The first check of `check_reg_arg` of Linux. + fn check_regno(&mut self, regno: u8) -> Result<()> { + if regno as usize >= MAX_BPF_REG { + verbose!(self, "R{} is invalid\n", regno); + return Err(Errno::EINVAL); + } + Ok(()) + } + fn do_insn(&mut self, pc: usize, st: &mut State) -> Result { let insn = self.insns[pc]; - if insn.dst_reg as usize >= MAX_BPF_REG || insn.src_reg as usize >= MAX_BPF_REG { - verbose!(self, "R{} is invalid\n", insn.dst_reg.max(insn.src_reg)); - return Err(Errno::EINVAL); + let class = bpf_class(insn.code); + // `check_alu_op` checks the reserved fields before the registers. The other classes + // check the registers first here, because they use the register numbers at once. + if !matches!(class, BPF_ALU | BPF_ALU64) { + self.check_regno(insn.dst_reg)?; + self.check_regno(insn.src_reg)?; } - match bpf_class(insn.code) { + match class { BPF_ALU | BPF_ALU64 => { self.check_alu_op(insn, st)?; Ok(Next::Go(pc + 1)) @@ -965,6 +1067,7 @@ impl Env<'_> { Access::Read, None, )?; + self.save_aux_ptr_type(pc, src.ty)?; st.regs[insn.dst_reg as usize] = if bpf_mode(insn.code) == BPF_MEMSX { Reg::unknown() } else { v }; Ok(Next::Go(pc + 1)) @@ -990,6 +1093,7 @@ impl Env<'_> { Access::Write, Some((insn.src_reg, src)), )?; + self.save_aux_ptr_type(pc, dst.ty)?; Ok(Next::Go(pc + 1)) } BPF_ST => { @@ -1013,9 +1117,14 @@ impl Env<'_> { Access::Write, Some((u8::MAX, v)), )?; + self.save_aux_ptr_type(pc, dst.ty)?; Ok(Next::Go(pc + 1)) } BPF_LD if insn.is_ld_imm64() => { + if insn.off != 0 { + verbose!(self, "BPF_LD_IMM64 uses reserved fields\n"); + return Err(Errno::EINVAL); + } self.check_dst(insn.dst_reg)?; let next = self.insns[pc + 1]; st.regs[insn.dst_reg as usize] = match insn.src_reg { @@ -1311,6 +1420,7 @@ impl Env<'_> { t: Access, val: Option<(u8, Reg)>, ) -> Result { + self.check_ptr_alignment(reg, off, size)?; if let Some((vregno, v)) = val && v.is_pointer() && !self.allow_ptr_leaks @@ -1405,6 +1515,49 @@ impl Env<'_> { } } + /// Keeps the pointer type of a load or a store, `save_aux_ptr_type` of Linux. An + /// instruction cannot access the context on one path and other memory on another path. + fn save_aux_ptr_type(&mut self, pc: usize, ty: RegType) -> Result<()> { + let prev = self.ptr_types[pc]; + if prev == RegType::NotInit { + self.ptr_types[pc] = ty; + } else if ty != prev && (ty == RegType::PtrToCtx || prev == RegType::PtrToCtx) { + verbose!(self, "same insn cannot be used with different pointers\n"); + return Err(Errno::EINVAL); + } + Ok(()) + } + + /// Checks the alignment of an access, `check_ptr_alignment` of Linux. An access of the stack + /// must always be aligned. An access of other memory must be aligned only with + /// `BPF_F_STRICT_ALIGNMENT`, or for an atomic instruction. + fn check_ptr_alignment(&mut self, reg: Reg, off: i16, size: u32) -> Result<()> { + let (desc, strict) = match reg.ty { + RegType::PtrToStack => ("stack ", true), + RegType::PtrToCtx => ("context ", self.strict_alignment), + RegType::PtrToMapValue => ("value ", self.strict_alignment), + _ => ("", self.strict_alignment), + }; + if !(strict || self.strict_once) || size == 1 { + return Ok(()); + } + let reg_off = reg.var.add(Tnum::constant((reg.off as i64 + off as i64) as u64)); + // `tnum_is_aligned`. + if (reg_off.value | reg_off.mask) & (u64::from(size) - 1) != 0 { + verbose!( + self, + "misaligned {}access off {}+{}+{} size {}\n", + desc, + TnumStr(reg.var), + reg.off, + off, + size + ); + return Err(Errno::EACCES); + } + Ok(()) + } + /// Checks an access of the stack and gives the offset from the frame pointer, as /// `check_stack_access_within_bounds` and `check_ptr_alignment` of Linux. fn check_stack_off(&mut self, regno: u8, reg: Reg, off: i16, size: u32) -> Result { @@ -1417,16 +1570,6 @@ impl Env<'_> { verbose!(self, "invalid stack R{} off={} size={}\n", regno, off, size); return Err(Errno::EACCES); } - if off.rem_euclid(size as i64) != 0 { - verbose!( - self, - "misaligned stack access off {}+{} size {}\n", - reg.off, - off - reg.off as i64, - size - ); - return Err(Errno::EACCES); - } Ok(off) } @@ -1444,8 +1587,10 @@ impl Env<'_> { } return Ok(Reg::unknown_of_size(size)); } + // A loader with `CAP_PERFMON` can read bytes that the program did not write, + // `allow_uninit_stack` of Linux. for b in start..start + size as usize { - if stack[b / 8].slot_type[b % 8] == STACK_INVALID { + if stack[b / 8].slot_type[b % 8] == STACK_INVALID && !self.allow_ptr_leaks { verbose!(self, "invalid read from stack off {}+{} size {}\n", off, b - start, size); return Err(Errno::EACCES); } @@ -1462,32 +1607,55 @@ impl Env<'_> { size: u32, value_size: u32, ) -> Result<()> { - let lo = reg.off as i64 + off as i64 + reg.umin as i64; - let hi = reg.off as i64 + off as i64 + reg.umax as i64; - if lo < 0 { - verbose!(self, "R{} min value is outside of the allowed memory range\n", regno); + // `check_mem_region_access` of Linux. + let off = reg.off as i64 + off as i64; + if reg.smin < 0 + && (reg.smin == i64::MIN + || off + reg.smin != i64::from((off + reg.smin) as i32) + || reg.smin + off < 0) + { verbose!( self, - "invalid access to map value, value_size={} off={} size={}\n", - value_size, - lo, - size + "R{} min value is negative, either use unsigned index or do a if (index >=0) check.\n", + regno ); return Err(Errno::EACCES); } - if hi + size as i64 > value_size as i64 { + if !self.check_mem_size(reg.smin + off, size, value_size) { + verbose!(self, "R{} min value is outside of the allowed memory range\n", regno); + return Err(Errno::EACCES); + } + if reg.umax >= BPF_MAX_VAR_OFF as u64 { verbose!( self, - "invalid access to map value, value_size={} off={} size={}\n", - value_size, - hi, - size + "R{} unbounded memory access, make sure to bounds check any such access\n", + regno ); return Err(Errno::EACCES); } + if !self.check_mem_size(reg.umax as i64 + off, size, value_size) { + verbose!(self, "R{} max value is outside of the allowed memory range\n", regno); + return Err(Errno::EACCES); + } Ok(()) } + /// Tells if an access of a map value is in the value, and logs the error if not, as + /// `__check_mem_access` of Linux. + fn check_mem_size(&mut self, off: i64, size: u32, value_size: u32) -> bool { + if off >= 0 && size > 0 && off as u64 + u64::from(size) <= u64::from(value_size) { + return true; + } + verbose!( + self, + "invalid access to map value, value_size={} off={} size={}\n", + value_size, + off as i32, + size + ); + false + } + /// The `is_valid_access` of the program type, as `bpf_verifier_ops` of Linux. fn is_valid_ctx_access(&self, off: i32, size: u32, t: Access) -> bool { if off < 0 || !(off as u32).is_multiple_of(size) { @@ -1495,8 +1663,10 @@ impl Env<'_> { } let off = off as u32; match self.prog_type { - BPF_PROG_TYPE_SOCKET_FILTER => skb_is_valid_access(off, size, t, false), - BPF_PROG_TYPE_CGROUP_SKB => skb_is_valid_access(off, size, t, true), + BPF_PROG_TYPE_SOCKET_FILTER => { + skb_is_valid_access(off, size, t, false, self.bpf_capable) + } + BPF_PROG_TYPE_CGROUP_SKB => skb_is_valid_access(off, size, t, true, self.bpf_capable), BPF_PROG_TYPE_CGROUP_SOCK => { sock_filter_is_valid_access(off, size, t, self.expected_attach_type) } @@ -1548,6 +1718,14 @@ impl Env<'_> { | BPF_PROG_TYPE_CGROUP_DEVICE => (0, 1), _ => return Ok(()), }; + if r0.is_pointer() { + verbose!( + self, + "At program exit the register R0 is not a known value ({})\n", + reg_type_str(r0.ty) + ); + return Err(Errno::EINVAL); + } // The 32-bit value is the result, as `return_32bit` of Linux. let (min, max) = if r0.umax <= u64::from(u32::MAX) { (r0.umin as i64, r0.umax as i64) @@ -1586,12 +1764,12 @@ impl Env<'_> { let dst = self.read_reg(st, insn.dst_reg)?; if imm == BPF_CMPXCHG { let r0 = self.read_reg(st, 0)?; - if r0.is_pointer() { + if r0.is_pointer() && !self.allow_ptr_leaks { verbose!(self, "R0 leaks addr into mem\n"); return Err(Errno::EACCES); } } - if src.is_pointer() { + if src.is_pointer() && !self.allow_ptr_leaks { verbose!(self, "R{} leaks addr into mem\n", insn.src_reg); return Err(Errno::EACCES); } @@ -1605,37 +1783,37 @@ impl Env<'_> { ); return Err(Errno::EACCES); } - RegType::PtrToMapValue => { - // The interpreter needs an aligned word for the atomic operation. - let total = dst.off as i64 + insn.off as i64; - if !dst.var.is_const() || total.rem_euclid(size as i64) != 0 { - verbose!( - self, - "misaligned value access off {}+{} size {}\n", - dst.off, - insn.off, - size - ); - return Err(Errno::EACCES); - } - } _ => {} } - self.check_mem_access(st, insn.dst_reg, dst, insn.off, size, Access::Read, None)?; - self.check_mem_access( - st, - insn.dst_reg, - dst, - insn.off, - size, - Access::Write, - Some((insn.src_reg, Reg::unknown_of_size(size))), - )?; - if imm == BPF_CMPXCHG { - st.regs[0] = Reg::unknown_of_size(size); + let load_reg = if imm == BPF_CMPXCHG { + Some(0) } else if imm & BPF_FETCH != 0 { self.check_dst(insn.src_reg)?; - st.regs[insn.src_reg as usize] = Reg::unknown_of_size(size); + Some(insn.src_reg) + } else { + None + }; + // The interpreter needs an aligned word for the atomic operation. A fetch gets the old + // value, which can be a pointer that the program spilled to the stack. + self.strict_once = true; + let old = self.check_mem_access(st, insn.dst_reg, dst, insn.off, size, Access::Read, None); + let r = old.and_then(|old| { + self.check_mem_access( + st, + insn.dst_reg, + dst, + insn.off, + size, + Access::Write, + Some((u8::MAX, Reg::unknown_of_size(size))), + )?; + Ok(old) + }); + self.strict_once = false; + if let Some(load_reg) = load_reg { + st.regs[load_reg as usize] = r?; + } else { + r?; } Ok(()) } @@ -1673,7 +1851,8 @@ impl Env<'_> { verbose!(self, "R0 !read_ok\n"); return Err(Errno::EACCES); } - if r0.is_pointer() { + // A loader with `CAP_PERFMON` can return a pointer, `is_pointer_value` of Linux. + if r0.is_pointer() && !self.allow_ptr_leaks { verbose!(self, "R0 leaks addr as return value\n"); return Err(Errno::EACCES); } @@ -1749,27 +1928,34 @@ impl Env<'_> { mark_ptr_or_null_regs(null_state, dst.id, true); mark_ptr_or_null_regs(ptr_state, dst.id, false); // `st` is the fall-through state and `other` the jump state. - return Ok(Next::Branch(pc + 1, target, alloc::boxed::Box::new(other))); + return Ok(Next::Branch(pc + 1, target, Box::new(other))); } if dst.is_pointer() || src.is_pointer() { if !self.allow_ptr_leaks { verbose!(self, "R{} pointer comparison prohibited\n", d); return Err(Errno::EACCES); } - return Ok(Next::Branch(pc + 1, target, alloc::boxed::Box::new(st.clone()))); + return Ok(Next::Branch(pc + 1, target, Box::new(st.clone()))); } - let (a, b) = if jmp32 { (zext32(dst), zext32(src)) } else { (dst, src) }; - let (a, b) = if jmp32 && !(a.is_const() && b.is_const()) { - (Reg::unknown(), Reg::unknown()) + // A comparison of 32 bits with a constant is a comparison of 64 bits when the upper + // half of `dst` follows from the lower half. Then the bounds of 64 bits refine as for + // JMP. Otherwise the verifier learns nothing. + let (a, b, both_const) = if !jmp32 { + (dst, src, false) + } else if dst.is_const() && src.is_const() { + (zext32(dst), zext32(src), true) } else { - (a, b) + match jmp32_as_jmp64(op, dst, src) { + Some(b) => (dst, b, false), + None => (Reg::unknown(), Reg::unknown(), false), + } }; - match is_branch_taken(op, a, b, jmp32) { + match is_branch_taken(op, a, b, both_const) { Some(true) => Ok(Next::Go(target)), Some(false) => Ok(Next::Go(pc + 1)), None => { let mut other = st.clone(); - if !jmp32 && b.is_const() { + if a == dst && b.is_const() { // `reg_set_min_max` of Linux, for a comparison with a constant. let taken = refine(op, dst, b.var.value, true); let not_taken = refine(op, dst, b.var.value, false); @@ -1789,7 +1975,7 @@ impl Env<'_> { (None, None) => return Ok(Next::Exit), } } - Ok(Next::Branch(pc + 1, target, alloc::boxed::Box::new(other))) + Ok(Next::Branch(pc + 1, target, Box::new(other))) } } } @@ -1806,18 +1992,17 @@ impl Env<'_> { return Err(Errno::E2BIG); } // The function gets R1 to R5 and a new frame pointer, as `set_callee_state`. - let mut regs = [Reg::NOT_INIT; MAX_BPF_REG]; + let mut regs = Box::new([Reg::NOT_INIT; MAX_BPF_REG]); regs[1..=5].copy_from_slice(&st.regs[1..=5]); regs[BPF_REG_FP as usize] = Reg { frameno: cur as u32 + 1, ..Reg::ptr(RegType::PtrToStack) }; // `clear_caller_saved_regs`. - let mut caller_regs = st.regs; + let mut caller_regs = core::mem::replace(&mut st.regs, regs); for r in caller_regs.iter_mut().take(6) { *r = Reg::NOT_INIT; } let stack = core::mem::replace(&mut st.stack, vec![StackSlot::EMPTY; SLOTS]); st.callers.push(Frame { regs: caller_regs, stack, subprog: st.subprog, callsite: pc }); - st.regs = regs; st.subprog = subprog; Ok(Next::Go(target)) } @@ -2089,7 +2274,8 @@ impl Env<'_> { let ok = match slot.slot_type[b % 8] { STACK_MISC => true, STACK_SPILL => slot.spilled.ty == RegType::Scalar || self.allow_ptr_leaks, - _ => false, + // `allow_uninit_stack` of Linux. + _ => self.allow_ptr_leaks, }; if !ok { verbose!( @@ -2142,7 +2328,7 @@ enum Next { /// The next instruction. Go(usize), /// Both ways of a jump: the first one now, the second one with its state later. - Branch(usize, usize, alloc::boxed::Box), + Branch(usize, usize, Box), /// The end of the path. Exit, } @@ -2407,11 +2593,47 @@ fn is_branch_taken(op: u8, a: Reg, b: Reg, jmp32: bool) -> Option { } } +/// The constant of 64 bits that `dst op src` of JMP compares with, when it gives the same result +/// as `dst op src` of JMP32. `src` must be a constant. When `dst` is a zero extension of its lower +/// 32 bits, the unsigned comparisons are the same with the zero extension of `src`. When `dst` is +/// a sign extension of its lower 32 bits, the signed comparisons are the same with the sign +/// extension of `src`. `BPF_JSET` is always the same with the zero extension. +fn jmp32_as_jmp64(op: u8, dst: Reg, src: Reg) -> Option { + if !src.is_const() || dst.ty != RegType::Scalar { + return None; + } + let lo = src.var.value as u32; + let zext = dst.umax <= u32::MAX as u64; + let sext = dst.smin >= i32::MIN as i64 && dst.smax <= i32::MAX as i64; + let v = match op { + BPF_JSET => lo as u64, + BPF_JGT | BPF_JGE | BPF_JLT | BPF_JLE if zext => lo as u64, + BPF_JSGT | BPF_JSGE | BPF_JSLT | BPF_JSLE if sext => lo as i32 as i64 as u64, + BPF_JEQ | BPF_JNE if zext => lo as u64, + BPF_JEQ | BPF_JNE if sext => lo as i32 as i64 as u64, + _ => return None, + }; + Some(Reg::known(v)) +} + /// The bounds of `r` when `r op v` is `taken`, as `reg_set_min_max` of Linux. `None` is a /// path that cannot occur. fn refine(op: u8, r: Reg, v: u64, taken: bool) -> Option { let mut r = r; let sv = v as i64; + if op == BPF_JSET { + // When the jump is taken, a bit of `v` is set, and the verifier knows which bit only + // when `v` has one bit. When it is not taken, all the bits of `v` are clear. + if taken { + if !v.is_power_of_two() { + return Some(r); + } + r.var = r.var.or(Tnum::constant(v)); + } else { + r.var = r.var.and(Tnum::constant(!v)); + } + return if r.bounds_sync() { Some(r) } else { None }; + } // A jump that is not taken is the opposite comparison. let op = if taken { op @@ -2497,34 +2719,66 @@ fn stack_write(stack: &mut [StackSlot], off: i64, size: u32, v: Reg) { } } -/// `bpf_skb_is_valid_access` of Linux with the rules of `sk_filter_is_valid_access` or, for -/// `cgroup` true, of `cg_skb_is_valid_access`. rukernel has no packet pointers in the verifier, -/// so `data` and `data_end` are not readable, as for a loader without `CAP_BPF`. `tstamp`, -/// `hwtstamp` and `sk` are not readable either. -fn skb_is_valid_access(off: u32, size: u32, t: Access, cgroup: bool) -> bool { +/// `sk_filter_is_valid_access` and `cg_skb_is_valid_access` of Linux, with +/// `bpf_skb_is_valid_access`. rukernel has no packet pointers and no `PTR_TO_SOCK_COMMON`, so a +/// program cannot read `data`, `data_end` or `sk`. +fn skb_is_valid_access(off: u32, size: u32, t: Access, cgroup: bool, bpf_capable: bool) -> bool { use sk_buff::*; - if off >= SIZE { + let within = |start: u32, len: u32| (start..start + len).contains(&off); + if cgroup { + if within(TC_CLASSID, 4) || within(DATA_META, 4) || within(WIRE_LEN, 4) { + return false; + } + if t == Access::Write + && !(within(MARK, 4) + || within(PRIORITY, 4) + || (CB..CB_END).contains(&off) + || (within(TSTAMP, 8) && bpf_capable)) + { + return false; + } + } else { + if within(TC_CLASSID, 4) + || within(DATA, 4) + || within(DATA_META, 4) + || within(DATA_END, 4) + || (FAMILY..LOCAL_PORT + 4).contains(&off) + || within(TSTAMP, 8) + || within(WIRE_LEN, 4) + || within(HWTSTAMP, 8) + { + return false; + } + if t == Access::Write && !(CB..CB_END).contains(&off) { + return false; + } + } + // `bpf_skb_is_valid_access`. + if off >= SIZE || !off.is_multiple_of(size) { return false; } if (CB..CB_END).contains(&off) { return off + size <= CB_END; } - let field = off & !3; - if t == Access::Write { - // A cgroup program can also write `mark` and `priority`. - return cgroup && matches!(field, MARK | PRIORITY) && size == 4; + if within(DATA, 4) || within(DATA_META, 4) || within(DATA_END, 4) || within(SK, 8) { + return false; + } + if (REMOTE_IP4..REMOTE_PORT).contains(&off) { + return size == 4; } - let readable = field < CB - || matches!(field, HASH | NAPI_ID | GSO_SEGS | GSO_SIZE) - || (cgroup && (FAMILY..=LOCAL_PORT).contains(&field)); - if !readable { + if within(FLOW_KEYS, 8) || off == TSTAMP_TYPE || (TSTAMP_TYPE + 1..HWTSTAMP).contains(&off) { return false; } - // The addresses can only be read whole. For the other fields a narrow read must be in the - // field, as `bpf_ctx_narrow_access_ok` of Linux. - if (REMOTE_IP4..REMOTE_PORT).contains(&field) { + if within(HWTSTAMP, 8) { + return t == Access::Read && size == 8; + } + if within(TSTAMP, 8) { + return size == 8; + } + if t == Access::Write { return size == 4; } + // `bpf_ctx_narrow_access_ok`. size <= 4 } @@ -2603,10 +2857,12 @@ fn sock_addr_is_valid_access(off: u32, size: u32, t: Access, attach_type: u32) - if !family_ok { return false; } - // An 8-byte access of an IPv6 address, `bpf_ctx_wide_access_ok` of Linux. + // An 8-byte access of an IPv6 address, `bpf_ctx_wide_access_ok` of Linux. The access must be + // aligned and in the address. The address `msg_src_ip6` ends at 60. + let in_field = |start: u32, end: u32| start <= off && off + size <= end; let wide = size == 8 && off.is_multiple_of(8) - && ((USER_IP6..USER_PORT).contains(&off) || (MSG_SRC_IP6..SK).contains(&off)); + && (in_field(USER_IP6, USER_PORT) || in_field(MSG_SRC_IP6, MSG_SRC_IP6 + 16)); match field { USER_IP4 | USER_PORT | MSG_SRC_IP4 => { if t == Access::Read { @@ -2675,6 +2931,7 @@ mod tests { expected_attach_type: 0, bpf_capable: caps, allow_ptr_leaks: caps, + strict_alignment: false, }; let mut log = BpfVerifierLog::new(1); let r = bpf_check( @@ -2908,9 +3165,47 @@ mod tests { #[test] fn loops_dead_code_and_bad_jumps_are_refused() { - let (r, log) = check(&[mov64_imm(0, 0), jmp_imm(BPF_JA, 0, 0, -2), exit()], true); + let (r, log) = check(&[mov64_imm(0, 0), jmp_imm(BPF_JA, 0, 0, -2), exit()], false); assert_eq!(r.err(), Some(Errno::EINVAL)); assert!(log.contains("back-edge from insn 1 to 0"), "{log}"); + // With `CAP_BPF` a loop is allowed, but it must end. + let (r, log) = check(&[mov64_imm(0, 0), jmp_imm(BPF_JEQ, 0, 0, -2), exit()], true); + assert_eq!(r.err(), Some(Errno::EINVAL)); + assert!(log.contains("infinite loop detected at insn 0"), "{log}"); + let p = [mov64_imm(0, 0), alu64_imm(BPF_ADD, 0, 1), jmp_imm(BPF_JNE, 0, 0, -2), exit()]; + let (r, log) = check(&p, true); + assert_eq!(r.err(), Some(Errno::E2BIG)); + assert!(log.contains("BPF program is too large. Processed 1000001 insn"), "{log}"); + // "jump test 4" of test_verifier: 2^36 paths with the same state. The pruning checks + // each instruction once. + let mut p = Vec::new(); + for i in 0..36 { + p.push(jmp_imm(BPF_JEQ, 1, 10, i % 4 + 1)); + } + p.extend([jmp_imm(BPF_JEQ, 1, 10, 0); 4]); + p.extend([mov64_imm(0, 0), exit()]); + let (r, log) = check(&p, true); + assert!(r.is_ok(), "{log}"); + let p = [mov64_imm(0, 0), alu64_imm(BPF_ADD, 0, 1), jmp_imm(BPF_JLT, 0, 10, -2), exit()]; + let (r, log) = check(&p, true); + assert!(r.is_ok(), "{log}"); + // The back edge of a branch that is never taken, from "dead code: start" of + // test_verifier. + let p = [ + jmp_imm(BPF_JA, 0, 0, 2), + BpfInsn { + code: BPF_LDX | BPF_MEM | BPF_B, + dst_reg: 8, + src_reg: 9, + ..BpfInsn::default() + }, + jmp_imm(BPF_JA, 0, 0, 2), + mov64_imm(0, 7), + jmp_imm(BPF_JGE, 0, 10, -4), + exit(), + ]; + let (r, log) = check(&p, true); + assert!(r.is_ok(), "{log}"); let (r, log) = check(&[mov64_imm(0, 0), exit(), exit()], true); assert_eq!(r.err(), Some(Errno::EINVAL)); assert!(log.contains("unreachable insn 2"), "{log}"); @@ -2927,9 +3222,11 @@ mod tests { let (r, log) = check(&[exit()], true); assert_eq!(r.err(), Some(Errno::EACCES)); assert!(log.contains("R0 !read_ok"), "{log}"); - let (r, log) = check(&[ldx_mem(BPF_DW, 0, BPF_REG_FP, -8), exit()], true); + let (r, log) = check(&[ldx_mem(BPF_DW, 0, BPF_REG_FP, -8), exit()], false); assert_eq!(r.err(), Some(Errno::EACCES)); assert!(log.contains("invalid read from stack"), "{log}"); + // With `CAP_PERFMON`, the read gives an unknown value. + assert!(check(&[ldx_mem(BPF_DW, 0, BPF_REG_FP, -8), exit()], true).0.is_ok()); let (r, log) = check(&[mov64_imm(BPF_REG_FP, 0), mov64_imm(0, 0), exit()], true); assert_eq!(r.err(), Some(Errno::EACCES)); assert!(log.contains("frame pointer is read only"), "{log}"); @@ -2998,9 +3295,10 @@ mod tests { exit(), ], ]); - let (r, log) = check(&p, true); + let (r, log) = check(&p, false); assert_eq!(r.err(), Some(Errno::EACCES)); assert!(log.contains("invalid indirect read from stack"), "{log}"); + assert!(check(&p, true).0.is_ok()); // A bad map descriptor. let p = concat(&[&ld_imm64(1, BPF_PSEUDO_MAP_FD, 9), &[mov64_imm(0, 0), exit()]]); let (r, log) = check(&p, true); @@ -3021,6 +3319,108 @@ mod tests { assert!(log.contains("invalid shift 64"), "{log}"); } + /// `BPF_RAND_UEXT_R7` of test_verifier: R7 is a random value of 31 bits. + fn rand_uext_r7() -> [BpfInsn; 4] { + [ + call(crate::helpers::func::BPF_FUNC_get_prandom_u32), + mov64_reg(7, 0), + alu64_imm(BPF_LSH, 7, 33), + alu64_imm(BPF_RSH, 7, 33), + ] + } + + /// The load from R9, which is not set, that test_verifier puts on a path that must be dead. + fn bad_load() -> BpfInsn { + ldx_mem(BPF_B, 8, 9, 0) + } + + fn jmp32_imm(op: u8, dst: u8, imm: i32, off: i16) -> BpfInsn { + BpfInsn { code: BPF_JMP32 | op | BPF_K, ..jmp_imm(op, dst, imm, off) } + } + + #[test] + fn jset_and_jmp32_refine_the_bounds() { + // "jset: range" of test_verifier. + let p = [ + call(crate::helpers::func::BPF_FUNC_get_prandom_u32), + mov64_reg(1, 0), + mov64_imm(0, 0), + alu64_imm(BPF_AND, 1, 0xff), + jmp_imm(BPF_JSET, 1, 0xf0, 3), + jmp_imm(BPF_JLT, 1, 0x10, 1), + bad_load(), + exit(), + jmp_imm(BPF_JSET, 1, 0x10, 1), + exit(), + jmp_imm(BPF_JGE, 1, 0x10, 1), + bad_load(), + exit(), + ]; + let (r, log) = check(&p, true); + assert!(r.is_ok(), "{log}"); + // "jset32: min/max deduction" and "jne32: min/max deduction". + let jset32 = concat(&[ + &rand_uext_r7(), + &[ + mov64_imm(0, 0), + jmp32_imm(BPF_JSET, 7, 0x10, 1), + exit(), + jmp32_imm(BPF_JGE, 7, 0x10, 1), + bad_load(), + exit(), + ], + ]); + let (r, log) = check(&jset32, true); + assert!(r.is_ok(), "{log}"); + let jne32 = concat(&[ + &rand_uext_r7(), + &[ + mov64_imm(0, 0), + jmp32_imm(BPF_JNE, 7, 0x10, 1), + jmp_imm(BPF_JNE, 7, 0x10, 1), + exit(), + bad_load(), + exit(), + ], + ]); + let (r, log) = check(&jne32, true); + assert!(r.is_ok(), "{log}"); + // The upper half of an unknown value can be set, so JMP32 tells nothing about JMP. + let p = [ + call(crate::helpers::func::BPF_FUNC_get_prandom_u32), + mov64_reg(7, 0), + alu64_imm(BPF_LSH, 7, 1), + mov64_imm(0, 0), + jmp32_imm(BPF_JNE, 7, 0x10, 1), + jmp_imm(BPF_JNE, 7, 0x10, 1), + exit(), + bad_load(), + exit(), + ]; + let (r, log) = check(&p, true); + assert_eq!(r.err(), Some(Errno::EACCES)); + assert!(log.contains("R9 !read_ok"), "{log}"); + } + + #[test] + fn the_order_of_the_errors_is_as_in_linux() { + // "junk insn5": the reserved fields of an ALU instruction come before the registers. + let junk = BpfInsn { code: 0x7f, dst_reg: 15, src_reg: 15, off: -1, imm: -1 }; + let (r, log) = check(&[junk, exit()], true); + assert_eq!(r.err(), Some(Errno::EINVAL)); + assert!(log.contains("BPF_ALU uses reserved fields"), "{log}"); + // "test1 ld_imm64": the log gives the first half of the 64-bit load. + let p = concat(&[ + &[jmp_imm(BPF_JEQ, 1, 0, 1)], + &ld_imm64(0, 0, 0), + &ld_imm64(0, 0, 1), + &[exit()], + ]); + let (r, log) = check(&p, true); + assert_eq!(r.err(), Some(Errno::EINVAL)); + assert!(log.contains("jump into the middle of ldimm64 insn 1"), "{log}"); + } + #[test] fn sock_addr_access() { use crate::prog::bpf_sock_addr::*; @@ -3031,6 +3431,10 @@ mod tests { assert!(!sock_addr_is_valid_access(USER_IP4, 2, Access::Write, v4)); assert!(!sock_addr_is_valid_access(USER_IP6, 4, Access::Read, v4)); assert!(sock_addr_is_valid_access(USER_IP6, 8, Access::Write, v6)); + let send6 = BPF_CGROUP_UDP6_SENDMSG; + assert!(sock_addr_is_valid_access(MSG_SRC_IP6 + 4, 8, Access::Read, send6)); + assert!(!sock_addr_is_valid_access(MSG_SRC_IP6 + 12, 8, Access::Read, send6)); + assert!(!sock_addr_is_valid_access(MSG_SRC_IP6 + 12, 8, Access::Write, send6)); assert!(!sock_addr_is_valid_access(USER_IP4, 4, Access::Read, v6)); assert!(sock_addr_is_valid_access(USER_PORT, 4, Access::Write, v6)); assert!(!sock_addr_is_valid_access(FAMILY, 4, Access::Write, v6)); @@ -3126,9 +3530,10 @@ mod tests { #[test] fn recursion_and_deep_calls_are_refused() { + // As on Linux, recursion stops at the limit of the frames. let (r, log) = check(&[call_local(1), exit(), call_local(-1), exit()], true); - assert_eq!(r.err(), Some(Errno::EINVAL)); - assert!(log.contains("back-edge from insn 2 to 2"), "{log}"); + assert_eq!(r.err(), Some(Errno::E2BIG)); + assert!(log.contains("the call stack of 9 frames is too deep"), "{log}"); // A chain of `n` functions after main. let chain = |n: usize| { let mut p = Vec::new(); diff --git a/core/rk-syscall/src/bpf.rs b/core/rk-syscall/src/bpf.rs index 20f970c..f7134af 100644 --- a/core/rk-syscall/src/bpf.rs +++ b/core/rk-syscall/src/bpf.rs @@ -85,6 +85,10 @@ impl Env for BpfEnv<'_, '_> { self.0.process().cred().uid } + fn helpers(&self) -> &dyn rk_bpf::helpers::HelperEnv { + &TaskHelperEnv + } + fn cgroup_get_from_fd(&self, fd: u32) -> Result { let file = self.fdget(fd as i32)?; let inode = file.inode().ok_or(Errno::EBADF)?; diff --git a/tests/user/init.c b/tests/user/init.c index 9781dee..3bef742 100644 --- a/tests/user/init.c +++ b/tests/user/init.c @@ -6379,10 +6379,15 @@ static void test_bpf_prog(void) errno = 0; ok = bpf_prog_new(insns, 11, log, 16) == -1 && errno == ENOSPC && strlen(log) == 15; check("a log that does not fit gives ENOSPC", ok); + /* + * A jump to itself. With CAP_BPF, check_cfg of Linux allows a back edge, and do_check finds + * the same state again at the jump. The NULL check is back, so that the store is valid. + */ + insns[6] = bpf_insn(0x15, 0, 0, 2, 0); insns[10] = bpf_insn(0x05, 0, 0, -1, 0); errno = 0; ok = bpf_prog_new(insns, 11, log, sizeof(log)) == -1 && errno == EINVAL && - strstr(log, "back-edge from insn 10 to 10"); + strstr(log, "infinite loop detected at insn 10"); check("the verifier refuses a loop", ok); close(map); } @@ -6427,11 +6432,14 @@ static void test_bpf_interp(void) close(sk[0]); close(sk[1]); close(prog); - /* A call to itself is a loop. */ + /* + * A call to itself. With CAP_BPF, check_cfg of Linux allows the back edge, and each call + * makes a new frame until the limit of 8 frames. + */ insns[10] = bpf_insn(0x85, 0, 1, 0, -2); errno = 0; - ok = bpf_prog_new(insns, 12, log, sizeof(log)) == -1 && errno == EINVAL && - strstr(log, "back-edge from insn 10 to 9"); + ok = bpf_prog_new(insns, 12, log, sizeof(log)) == -1 && errno == E2BIG && + strstr(log, "the call stack of 9 frames is too deep"); check("the verifier refuses recursion", ok); close(map); } diff --git a/tools/kselftest/bpf.md b/tools/kselftest/bpf.md new file mode 100644 index 0000000..bf5fd49 --- /dev/null +++ b/tools/kselftest/bpf.md @@ -0,0 +1,34 @@ +# kselftest test_verifier on rukernel + +This file gives the result of bpf/test_verifier of the Linux selftests on rukernel, for the verifier of issue #537. tools/kselftest/build.sh builds test_verifier from Linux v6.17 with a static libbpf, and `cargo xtask boot-test kselftest` runs it as root in the initramfs of the test. + +## Scope + +Each test of test_verifier has a program, a program type, and the result and the verifier message of Linux. A test passes when rukernel accepts or refuses the program as Linux does, and gives the same message. For a test that Linux accepts, test_verifier also runs the program with `BPF_PROG_TEST_RUN`, and the return value must be the value of the test. + +The verifier of rukernel is for the program types of issue #537: `SOCKET_FILTER`, `CGROUP_SKB`, `CGROUP_SOCK`, `CGROUP_SOCK_ADDR` and `CGROUP_DEVICE`. build.sh adds a check to test_verifier that skips the tests of the other program types, with the message `SKIP (program type N)`. + +## Result + +Run of 2026-11-11, rukernel debug build, x86-64, QEMU TCG with one CPU: + +``` +Summary: 281 PASSED, 510 SKIPPED, 0 FAILED +``` + +All 281 tests of the five program types that run give the result of Linux. The first run, before the fixes of the verifier PR, gave 259 PASSED and 22 FAILED. + +The skipped tests: + +| Count | Reason | +| --- | --- | +| 264 | The unprivileged run (`#N/u`) of each test. The kernel has `kernel.unprivileged_bpf_disabled=2`, so test_verifier runs only as root (`#N/p`). | +| 237 | `SKIP (program type N)`: the program types that issue #537 does not cover. These are 122 tests of `SCHED_CLS` (3), 28 of `TRACEPOINT` (5), 25 of `XDP` (6), 25 of `SK_LOOKUP` (30), 18 of `SK_SKB` (14), 6 of `TRACING` (26), 4 of `PERF_EVENT` (7), 2 of `SCHED_ACT` (4), and 1 each of `KPROBE` (2), `LWT_OUT` (11), `SOCK_OPS` (13), `CGROUP_SYSCTL` (23), `CGROUP_SOCKOPT` (25), `EXT` (28) and `LSM` (29). | +| 6 | `SKIP (requires BPF JIT)`: the inline of `bpf_loop`. rukernel has no JIT before R7. | +| 3 | `SKIP (unsupported map type 4)`: `BPF_MAP_TYPE_PERF_EVENT_ARRAY` is not in issue #537. | + +## Notes + +- rukernel cannot load bpf_testmod.ko. build.sh changes test_verifier so that it runs on without the module. No test of the five program types fails without the module. +- The verifier prunes a path when its state is equal to a state that it saw before at the same jump target, as `is_state_visited` of Linux. A path that comes back to an equal state is a loop that does not end, and the verifier gives "infinite loop detected at insn N". A loop that changes its state goes on to the limit of 1000000 instructions and gives "BPF program is too large". +- The registers and the stack of a verifier state are on the heap. A debug build of the kernel keeps many copies of a state on the stack, and the 32 KiB kernel stack overflowed with large states. diff --git a/tools/kselftest/build.sh b/tools/kselftest/build.sh index c1a28ff..d87045d 100755 --- a/tools/kselftest/build.sh +++ b/tools/kselftest/build.sh @@ -4,8 +4,9 @@ # Usage: tools/kselftest/build.sh OUT_DIR # # The script needs docker. It takes the selftests of the Linux release in LINUX_TAG, and builds -# only the tests in TESTS. The test programs go to OUT_DIR. `cargo xtask boot-test` finds them in -# target/kselftest, or in the directory that RK_KSELFTEST gives. +# only the tests in TESTS, and bpf/test_verifier with a static libbpf. The test programs go to +# OUT_DIR. `cargo xtask boot-test` finds them in target/kselftest, or in the directory that +# RK_KSELFTEST gives. # # The Python tests of tpm2 go to OUT_DIR/tpm2. The `kselftest` boot test does not run them: the # `tpm` boot test runs them with the python3 of tools/tpm/build.sh, on x86-64 and on arm64. @@ -27,11 +28,11 @@ docker run --rm --platform linux/amd64 -v "$out:/out" -e LINUX_TAG="$LINUX_TAG" debian:trixie sh -euc ' apt-get update -qq >/dev/null apt-get install -qq -y --no-install-recommends gcc libc6-dev libcap-dev linux-libc-dev git \ - ca-certificates >/dev/null + ca-certificates make python3 libelf-dev zlib1g-dev libzstd-dev >/dev/null git clone --quiet --depth 1 --branch "$LINUX_TAG" --filter=blob:none --sparse \ https://github.com/torvalds/linux /linux cd /linux -git sparse-checkout set --no-cone /tools/testing/selftests/ >/dev/null +git sparse-checkout set --no-cone /tools/ /kernel/bpf/disasm.c /kernel/bpf/disasm.h >/dev/null cd tools/testing/selftests for t in $TESTS; do # The tests of lsm also need lsm/common.c, as their Makefile. @@ -52,6 +53,37 @@ done mkdir -p /out/tpm2 cp tpm2/test_smoke.sh tpm2/test_space.sh tpm2/test_async.sh tpm2/tpm2.py tpm2/tpm2_tests.py \ /out/tpm2/ +# bpf/test_verifier, as its rule in the Makefile of selftests/bpf, with a static libbpf. +mkdir -p /b/libbpf +make -s -C /linux/tools/lib/bpf -j"$(nproc)" OUTPUT=/b/libbpf/ BUILD_STATIC_ONLY=1 \ + DESTDIR=/b/inst prefix=/usr install >/dev/null +cd bpf +# rukernel cannot load bpf_testmod.ko, the module of the tests. Without the module, only the +# tests of its kernel functions fail, so the test runs on. +sed -i "/if (load_bpf_testmod(verbose))/{N;s/.*\n.*/\tload_bpf_testmod(verbose);/}" test_verifier.c +# The verifier of rukernel is for the program types of issue #537 only, so the other tests do not +# apply. tools/kselftest/bpf.md gives the result. +sed -i "/^\tfixup_skips = skips;/i\\ +\tif (prog_type != BPF_PROG_TYPE_SOCKET_FILTER \&\& prog_type != BPF_PROG_TYPE_CGROUP_SKB \&\&\\ +\t prog_type != BPF_PROG_TYPE_CGROUP_SOCK \&\&\\ +\t prog_type != BPF_PROG_TYPE_CGROUP_SOCK_ADDR \&\&\\ +\t prog_type != BPF_PROG_TYPE_CGROUP_DEVICE) {\\ +\t\tprintf(\"SKIP (program type %d)\\\\n\", prog_type);\\ +\t\tskips++;\\ +\t\treturn;\\ +\t}" test_verifier.c +(cd verifier && echo "#ifdef FILL_ARRAY" && ls *.c | sed "s/.*/#include \"&\"/" && + echo "#endif") > verifier/tests.h +# The libelf.a of Debian trixie does not have the search tree of elfutils. +printf "%s\n" "#include " "struct t { void *root; };" \ + "void eu_search_tree_init(struct t *t) { t->root = 0; }" \ + "void eu_search_tree_fini(struct t *t, void (*f)(void *))" \ + "{ if (t->root) tdestroy(t->root, f); t->root = 0; }" > /b/eu-search.c +gcc -static -O2 -std=gnu11 -D_GNU_SOURCE -I. -I/b/inst/usr/include -I/linux/tools/include \ + -I/linux/tools/arch/x86/include -I/linux/tools/include/uapi -o /out/test_verifier \ + test_verifier.c testing_helpers.c cap_helpers.c unpriv_helpers.c disasm.c /b/eu-search.c \ + /b/inst/usr/lib64/libbpf.a -lelf -lz -lzstd -lrt -lpthread +strip /out/test_verifier echo "$LINUX_TAG" > /out/VERSION ' echo "kselftest: the tests are in $out"