diff --git a/Cargo.lock b/Cargo.lock index 164483f2..3841840e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,7 +787,6 @@ dependencies = [ "bindgen", "bitfield-struct 0.11.0", "bitint", - "blake3", "bytemuck", "cc", "chrono", @@ -815,6 +814,7 @@ name = "fixhandle" version = "0.1.0" dependencies = [ "bitint", + "blake3", "common", "derive_more", ] diff --git a/common/src/protocol/control.rs b/common/src/protocol/control.rs index a6d315cf..c4f738d3 100644 --- a/common/src/protocol/control.rs +++ b/common/src/protocol/control.rs @@ -9,6 +9,7 @@ pub use embedded_io::ErrorKind; #[derive(Debug, Serialize, Deserialize)] pub enum Request { GetArgs, + CurrentDir, Exit(i32), Open(String, FileMode), Mkdir(String), @@ -19,6 +20,7 @@ pub enum Request { #[derive(Debug, Serialize, Deserialize)] pub enum Response { Args(Vec), + CurrentDir(String), Pipe(PipeData), Ack, Err(IoErrorKind), diff --git a/fix/Cargo.toml b/fix/Cargo.toml index 6572031c..8b67d137 100644 --- a/fix/Cargo.toml +++ b/fix/Cargo.toml @@ -34,7 +34,6 @@ user = { path = "../user", artifact = "bin", target = "x86_64-unknown-none" } async-lock = { version = "3.4.1", default-features = false } bytemuck = "1.24.0" bitfield-struct = "0.11.0" -blake3 = { version = "1.8.5", default-features = false } hex = { version = "0.4.3", default-features = false, features = ["alloc"] } bitint = "0.1.1" crossbeam-queue = { diff --git a/fix/handle/Cargo.toml b/fix/handle/Cargo.toml index d93ad375..a1a04573 100644 --- a/fix/handle/Cargo.toml +++ b/fix/handle/Cargo.toml @@ -9,6 +9,7 @@ edition = "2024" derive_more = { version = "2.0.1", default-features = false, features = ["full"] } common = { path = "../../common", default-features = false } bitint = "0.1.1" +blake3 = { version = "1.8.5", default-features = false } [features] testing-mode = [] diff --git a/fix/handle/src/lib.rs b/fix/handle/src/lib.rs index 7a199523..89f54e32 100644 --- a/fix/handle/src/lib.rs +++ b/fix/handle/src/lib.rs @@ -7,6 +7,17 @@ use bitint::U5; pub use common::bitpack::BitPack; use derive_more::{From, Into, TryUnwrap, Unwrap}; +/// Return the storage-independent Fix name of serialized content. +/// +/// This does not construct a handle; handle construction stays in Fix's +/// existing Storage/FixOnArca operation path. +pub fn canonicalize(data: &[u8]) -> [u8; 24] { + let hash = blake3::hash(data); + let mut name = [0u8; 24]; + name.copy_from_slice(&hash.as_bytes()[..24]); + name +} + const fn bitmask256() -> [u8; 32] { assert!(I + WIDTH <= 256); let mut out = [0u8; 32]; @@ -47,6 +58,23 @@ impl Handle { pub fn is_empty(&self) -> bool { self.len() == 0 } + + pub fn is_literal(&self) -> bool { + matches!(self, Handle::Object(Object::Blob(Blob::Literal(_)))) + } + + pub fn is_canonical(&self) -> bool { + match self { + Handle::Ref(x) => x.is_canonical(), + Handle::Object(x) => x.is_canonical(), + Handle::Thunk(x) => x.is_canonical(), + Handle::Encode(x) => x.is_canonical(), + } + } + + pub fn is_machine(&self) -> bool { + !self.is_canonical() + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, TryUnwrap, Unwrap, From)] @@ -67,6 +95,13 @@ impl Ref { pub fn is_empty(&self) -> bool { self.len() == 0 } + + fn is_canonical(&self) -> bool { + match self { + Ref::Blob(x) => x.is_canonical(), + Ref::Tree(x) => x.is_canonical(), + } + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, TryUnwrap, Unwrap, From)] @@ -87,6 +122,13 @@ impl Object { pub fn is_empty(&self) -> bool { self.len() == 0 } + + fn is_canonical(&self) -> bool { + match self { + Object::Blob(x) => x.is_canonical(), + Object::Tree(x) => x.is_canonical(), + } + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, Unwrap)] @@ -108,6 +150,13 @@ impl Thunk { pub fn is_empty(&self) -> bool { self.len() == 0 } + + fn is_canonical(&self) -> bool { + match self { + Thunk::Identification(x) => x.is_canonical(), + Thunk::Application(x) | Thunk::Selection(x) => x.is_canonical(), + } + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, TryUnwrap, Unwrap)] @@ -128,6 +177,12 @@ impl Encode { pub fn is_empty(&self) -> bool { self.len() == 0 } + + fn is_canonical(&self) -> bool { + match self { + Encode::Strict(x) | Encode::Shallow(x) => x.is_canonical(), + } + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, TryUnwrap, Unwrap)] @@ -148,6 +203,12 @@ impl Tree { pub fn is_empty(&self) -> bool { self.len() == 0 } + + fn is_canonical(&self) -> bool { + match self { + Tree::Tree(x) | Tree::Tag(x) => x.is_canonical(), + } + } } #[derive(BitPack, Debug, Copy, Clone, Eq, PartialEq, TryUnwrap, Unwrap)] @@ -168,6 +229,17 @@ impl Blob { pub fn is_empty(&self) -> bool { self.len() == 0 } + + pub fn is_literal(&self) -> bool { + matches!(self, Blob::Literal(_)) + } + + fn is_canonical(&self) -> bool { + match self { + Blob::Blob(x) => x.is_canonical(), + Blob::Literal(_) => true, + } + } } #[derive(Debug, Copy, Clone, Eq, PartialEq, From, Into)] @@ -200,6 +272,14 @@ impl BlobName { pub fn is_empty(&self) -> bool { self.len() == 0 } + + pub fn is_canonical(&self) -> bool { + self.0.is_canonical() + } + + pub fn is_machine(&self) -> bool { + !self.is_canonical() + } } impl LiteralName { @@ -243,6 +323,14 @@ impl TreeName { pub fn is_empty(&self) -> bool { self.len() == 0 } + + pub fn is_canonical(&self) -> bool { + self.0.is_canonical() + } + + pub fn is_machine(&self) -> bool { + !self.is_canonical() + } } impl common::bitpack::BitPack for BlobName { @@ -336,6 +424,13 @@ pub struct RawName { } impl RawName { + /// The first high bit left unused by the complete [`Handle`] tag marks a + /// named handle canonical. Zero remains machine-compatible with the + /// original `MemoryStorage`; process-local names do not cross this ABI. + pub const ADDRESS_SHIFT: u32 = Handle::TAGBITS - BlobName::TAGBITS; + pub const MACHINE_NAME: u16 = 0; + pub const CANONICAL_NAME: u16 = 1 << Self::ADDRESS_SHIFT; + pub fn forge(bytes: [u8; 32]) -> Self { let mut name = [0; 24]; name.copy_from_slice(&bytes[..24]); @@ -357,6 +452,10 @@ impl RawName { bytes[30..32].copy_from_slice(&self.meta.to_le_bytes()); bytes } + + pub fn is_canonical(&self) -> bool { + self.meta & Self::CANONICAL_NAME != 0 + } } impl From for [u8; 32] { diff --git a/fix/src/evaluator.rs b/fix/src/evaluator.rs index 0a62aaa8..cd94482b 100644 --- a/fix/src/evaluator.rs +++ b/fix/src/evaluator.rs @@ -91,7 +91,10 @@ impl Evaluator { .copied() .map(|x| self.eval(x)) .collect(); - self.runtime.storage().add_tree(&evaled) + self.runtime + .storage() + .add_tree(&evaled) + .expect("storage failed to create tree") } pub fn eval(&self, handle: Handle) -> Handle { diff --git a/fix/src/interpreter/interpreter.rs b/fix/src/interpreter/interpreter.rs index a39646a9..c28472bf 100644 --- a/fix/src/interpreter/interpreter.rs +++ b/fix/src/interpreter/interpreter.rs @@ -50,11 +50,17 @@ impl FixShell for Interpreter<'_> { type Handle = Handle; fn create_blob(&self, data: &[u8]) -> Self::Handle { - self.storage.add_blob(data).into() + self.storage + .add_blob(data) + .expect("storage failed to create blob") + .into() } fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle { - self.storage.add_tree(data).into() + self.storage + .add_tree(data) + .expect("storage failed to create tree") + .into() } fn create_ref(handle: Self::Handle) -> Self::Handle { diff --git a/fix/src/main.rs b/fix/src/main.rs index 2824c99b..28c5bf15 100644 --- a/fix/src/main.rs +++ b/fix/src/main.rs @@ -6,12 +6,14 @@ mod parallel_evaluator; mod scheduler; -use kernel::host::fs; +use kernel::host::fs::{File, Whence}; use kernel::host::os; use kernel::prelude::*; use fix::arca::FixOnArca; use fix::parser::*; +use fix::runtime::Runtime; +use fix::storage::disk::DiskStorage; use fix::*; #[cfg(test)] @@ -28,9 +30,13 @@ fn tests() { fn main() { let argv = os::argv(); - // Subcommand dispatch: `fix init` | `fix eval `. + // Subcommand dispatch: `fix init` | `fix create-blob ` | `fix eval `. match argv.get(1).map(String::as_str) { Some("init") => init(), + Some("create-blob") => { + let filename = argv.get(2).expect("fix create-blob: expected a file"); + create_blob(filename); + } Some("eval") => { let path = argv.get(2).expect("fix eval: expected a command file"); eval_file(path) @@ -40,24 +46,64 @@ fn main() { let path = argv.get(2).expect("fix eval: expected a command file"); eval_file_parallel(path); } - Some(other) => panic!("fix: unknown command '{other}' (expected: init | eval )"), - None => panic!("fix: expected a command (init | eval "), + Some(other) => panic!( + "fix: unknown command '{other}' (expected: init | create-blob | eval | parallel_eval )" + ), + None => panic!( + "fix: expected a command (init | create-blob | eval | parallel_eval )" + ), } kernel::shutdown(); } -/// `fix init`: create the on-disk `.fix` store with its `objects/` and -/// `labels/` subdirs. `mkdir` maps to host `create_dir_all`, so re-running on an -/// existing store is harmless (matches git's "reinitialized existing repository"). +/// `fix init`: initialize the on-disk `.fix` store. fn init() { - for dir in [".fix/objects", ".fix/labels"] { - if let Err(e) = fs::mkdir(dir) { - println!("fix init: failed to create {dir}: {e:?}"); + if let Err(error) = DiskStorage::try_new() { + println!("fix init: failed to initialize DiskStorage: {error:?}"); + kernel::exit(1); + } + let current_dir = match os::current_dir() { + Ok(path) => path, + Err(error) => { + println!("fix init: cannot resolve the current directory: {error:?}"); + kernel::exit(1); + } + }; + if current_dir == "/" { + println!("initialized empty fix store in /.fix"); + } else { + println!("initialized empty fix store in {current_dir}/.fix"); + } +} + +/// `fix create-blob `: content-address the file's bytes, persist an out-of-line +/// blob under `.fix/objects`, and print its canonical handle. Small blobs are +/// represented directly by an inline literal handle and require no object file. +fn create_blob(filename: &str) { + let mut file = File::open(filename, true, false, false, false, false) + .unwrap_or_else(|e| panic!("fix create-blob: cannot open {filename}: {e:?}")); + let len = file.seek(Whence::End(0)) as usize; + file.seek(Whence::Start(0)); + let mut buf = vec![0; len]; + file.read_exact(&mut buf); + + let source: FixOnArca = FixOnArca::default(); + let machine = match source.storage().add_blob(&buf) { + Ok(blob) => Handle::from(blob), + Err(error) => { + println!("fix create-blob: cannot create machine blob: {error:?}"); + kernel::exit(1); + } + }; + let destination = DiskStorage; + match destination.import(source.storage(), machine) { + Ok(canonical) => println!("{canonical}"), + Err(error) => { + println!("fix create-blob: export failed: {error:?}"); kernel::exit(1); } } - println!("initialized empty fix store in .fix"); } // Jennifer: tons of redundancy but I just didn't want to change original code, diff --git a/fix/src/parallel_evaluator.rs b/fix/src/parallel_evaluator.rs index e1886b65..8a4d0a75 100644 --- a/fix/src/parallel_evaluator.rs +++ b/fix/src/parallel_evaluator.rs @@ -145,7 +145,10 @@ impl Evaluator { .copied() .map(|x| self.eval_test(x, EvalType::Serial)) .collect(); - self.runtime.storage().add_tree(&evaled) + self.runtime + .storage() + .add_tree(&evaled) + .expect("storage failed to create tree") } fn eval_tree_parallel(&self, handle: Tree) -> Tree { @@ -163,7 +166,10 @@ impl Evaluator { for task in tasks { evaled.push(self.wait_while_helping(&task)); } - self.runtime.storage().add_tree(&evaled) + self.runtime + .storage() + .add_tree(&evaled) + .expect("storage failed to create tree") } // elimnate redundancy i think fn eval_test(&self, handle: Handle, eval_mode: EvalType) -> Handle { diff --git a/fix/src/runtime/arca.rs b/fix/src/runtime/arca.rs index 098b724c..748f87ce 100644 --- a/fix/src/runtime/arca.rs +++ b/fix/src/runtime/arca.rs @@ -61,7 +61,9 @@ impl FixOnArca { panic!() }; k.apply(pack_handle( - self.storage().add_blob(&u32::to_le_bytes(w.read() as u32)), + self.storage() + .add_blob(&u32::to_le_bytes(w.read() as u32)) + .expect("storage failed to create blob"), )) } b"create_blob_i64" => { @@ -69,14 +71,20 @@ impl FixOnArca { panic!() }; k.apply(pack_handle( - self.storage().add_blob(&u64::to_le_bytes(w.read())), + self.storage() + .add_blob(&u64::to_le_bytes(w.read())) + .expect("storage failed to create blob"), )) } b"create_blob" => { let Some(Value::Blob(b)) = args.pop() else { panic!() }; - k.apply(pack_handle(self.storage().add_blob(&b))) + k.apply(pack_handle( + self.storage() + .add_blob(&b) + .expect("storage failed to create blob"), + )) } b"create_tree" => { let Some(Value::Blob(t)) = args.pop() else { @@ -86,7 +94,11 @@ impl FixOnArca { for handle in t.chunks(32) { tree.push(Handle::unpack(handle.try_into().unwrap())); } - k.apply(pack_handle(self.storage().add_tree(&tree))) + k.apply(pack_handle( + self.storage() + .add_tree(&tree) + .expect("storage failed to create tree"), + )) } b"get_blob" => { let Some(Value::Blob(b)) = args.pop() else { diff --git a/fix/src/storage.rs b/fix/src/storage.rs index 4cd48a21..cec3e23d 100644 --- a/fix/src/storage.rs +++ b/fix/src/storage.rs @@ -2,18 +2,43 @@ extern crate alloc; use super::*; use alloc::boxed::Box; +use common::protocol::control::ErrorKind; use core::option::Option; +pub mod disk; pub mod memory; +#[derive(Debug, Copy, Clone)] +pub struct StorageError(pub ErrorKind); + +impl From for StorageError { + fn from(value: ErrorKind) -> Self { + Self(value) + } +} + +#[derive(Debug)] +pub enum ImportError { + Unresolved(Handle), + Storage(StorageError), +} + +impl From for ImportError { + fn from(value: StorageError) -> Self { + Self::Storage(value) + } +} + /// An object store, capable of saving and retrieving Fix objects. pub trait Storage { - fn add_blob(&self, data: &[u8]) -> Blob; - fn add_tree(&self, data: &[Handle]) -> Tree; + fn add_blob(&self, data: &[u8]) -> Result; + fn add_tree(&self, data: &[Handle]) -> Result; fn get_blob(&self, name: Blob) -> Option>; fn get_tree(&self, name: Tree) -> Option>; + fn import(&self, from: &dyn Storage, handle: Handle) -> Result; + fn has_blob(&self, name: Blob) -> bool { self.get_blob(name).is_some() } @@ -21,4 +46,11 @@ pub trait Storage { fn has_tree(&self, name: Tree) -> bool { self.get_tree(name).is_some() } + + fn export(&self, handle: Handle, to: &dyn Storage) -> Result + where + Self: Sized, + { + to.import(self, handle) + } } diff --git a/fix/src/storage/disk.rs b/fix/src/storage/disk.rs new file mode 100644 index 00000000..66f784c3 --- /dev/null +++ b/fix/src/storage/disk.rs @@ -0,0 +1,111 @@ +extern crate alloc; + +use super::*; +use alloc::boxed::Box; +use alloc::format; +use alloc::vec; +use bitint::U48; +use common::protocol::control::ErrorKind; +use fixhandle::canonicalize; +use kernel::host::fs::{self, File}; + +const OBJECTS_DIR: &str = ".fix/objects"; +const LABELS_DIR: &str = ".fix/labels"; + +/// Canonical Fix storage backed by the existing host filesystem interface. +/// Import establishes the semantic invariants; this backend makes persistence +/// failures and pre-existing corruption observable to that operation. +#[derive(Debug)] +pub struct DiskStorage; + +impl DiskStorage { + pub fn try_new() -> Result { + fs::mkdir(OBJECTS_DIR)?; + fs::mkdir(LABELS_DIR)?; + Ok(Self) + } + + fn read_object(handle: Handle, expected_len: usize) -> Result, StorageError> { + let path = format!("{OBJECTS_DIR}/{handle}"); + let mut file = File::open(&path, true, false, false, false, false)?; + let mut data = vec![0; expected_len]; + if file.read_exact(&mut data) != expected_len { + return Err(ErrorKind::InvalidData.into()); + } + + let mut extra = [0]; + if file.read(&mut extra) != 0 { + return Err(ErrorKind::InvalidData.into()); + } + Ok(data.into()) + } + + fn write_object(handle: Handle, data: &[u8]) -> Result<(), StorageError> { + match Self::read_object(handle, data.len()) { + Ok(existing) if existing.as_ref() == data => Ok(()), + Ok(_) => Err(ErrorKind::InvalidData.into()), + Err(StorageError(ErrorKind::NotFound)) => { + let path = format!("{OBJECTS_DIR}/{handle}"); + let mut file = File::open(&path, false, true, true, false, true)?; + if file.write_exact(data) != data.len() { + return Err(ErrorKind::InvalidData.into()); + } + drop(file); + + let persisted = Self::read_object(handle, data.len())?; + if persisted.as_ref() != data { + return Err(ErrorKind::InvalidData.into()); + } + Ok(()) + } + Err(error) => Err(error), + } + } +} + +impl Storage for DiskStorage { + fn add_blob(&self, data: &[u8]) -> Result { + if data.len() < 30 { + return Ok(Blob::Literal(LiteralName::new(data))); + } + + let blob = unsafe { + BlobName::new(RawName { + name: canonicalize(data), + size: U48::new(data.len() as u64).expect("blob larger than 2^48 bytes"), + meta: RawName::CANONICAL_NAME, + }) + .into() + }; + Self::write_object(Handle::from(blob), data)?; + Ok(blob) + } + + fn add_tree(&self, _data: &[Handle]) -> Result { + todo!("DiskStorage::add_tree is not implemented") + } + + fn get_blob(&self, name: Blob) -> Option> { + match name { + Blob::Literal(literal) => Some(literal.bytes().into()), + Blob::Blob(blob) => Self::read_object(Handle::from(Blob::Blob(blob)), blob.len()).ok(), + } + } + + fn get_tree(&self, _name: Tree) -> Option> { + todo!("DiskStorage::get_tree is not implemented") + } + + fn import(&self, from: &dyn Storage, handle: Handle) -> Result { + match handle { + Handle::Object(Object::Blob(blob)) if blob.is_literal() => Ok(handle), + Handle::Object(Object::Blob(blob)) => { + let bytes = from + .get_blob(blob) + .ok_or_else(|| ImportError::Unresolved(Handle::from(blob)))?; + Ok(Handle::from(self.add_blob(&bytes)?)) + } + _ => todo!("DiskStorage::import only supports Blob objects"), + } + } +} diff --git a/fix/src/storage/memory.rs b/fix/src/storage/memory.rs index 1c46d7ad..e806a89e 100644 --- a/fix/src/storage/memory.rs +++ b/fix/src/storage/memory.rs @@ -15,41 +15,41 @@ pub struct MemoryStorage { } impl Storage for MemoryStorage { - fn add_blob(&self, data: &[u8]) -> Blob { + fn add_blob(&self, data: &[u8]) -> Result { let mut blobs = self.blobs.lock(); let i = blobs.len(); let len = data.len(); if len < 30 { - return Blob::Literal(LiteralName::new(data)); + return Ok(Blob::Literal(LiteralName::new(data))); } blobs.push(data.into()); let mut name = [0; 24]; name[0..8].copy_from_slice(&usize::to_le_bytes(!i)); - unsafe { + Ok(unsafe { BlobName::new(RawName { name, size: U48::new(len as u64).unwrap(), - meta: 0, + meta: RawName::MACHINE_NAME, }) .into() - } + }) } - fn add_tree(&self, data: &[Handle]) -> Tree { + fn add_tree(&self, data: &[Handle]) -> Result { let mut trees = self.trees.lock(); let i = trees.len(); let len = data.len(); trees.push(data.into()); let mut name = [0; 24]; name[0..8].copy_from_slice(&usize::to_le_bytes(!i)); - unsafe { + Ok(unsafe { TreeName::new(RawName { name, size: U48::new(len as u64).unwrap(), - meta: 0, + meta: RawName::MACHINE_NAME, }) .into() - } + }) } fn get_blob(&self, name: Blob) -> Option> { @@ -59,7 +59,10 @@ impl Storage for MemoryStorage { Blob::Blob(name) => name, Blob::Literal(name) => return Some(name.bytes().into()), }; - i.copy_from_slice(&BlobName::from(name).name().name[0..8]); + if !name.is_machine() { + return None; + } + i.copy_from_slice(&name.name().name[0..8]); let i = !usize::from_le_bytes(i); blobs.get(i).cloned() } @@ -67,8 +70,16 @@ impl Storage for MemoryStorage { fn get_tree(&self, name: Tree) -> Option> { let trees = self.trees.lock(); let mut i = [0; 8]; - i.copy_from_slice(&TreeName::from(name).name().name[0..8]); + let name = TreeName::from(name); + if !name.is_machine() { + return None; + } + i.copy_from_slice(&name.name().name[0..8]); let i = !usize::from_le_bytes(i); trees.get(i).cloned() } + + fn import(&self, _from: &dyn Storage, _handle: Handle) -> Result { + panic!("MemoryStorage::import is not implemented") + } } diff --git a/kernel/src/host.rs b/kernel/src/host.rs index e43a7fb9..dcfc1dcc 100644 --- a/kernel/src/host.rs +++ b/kernel/src/host.rs @@ -104,6 +104,16 @@ pub mod os { }; args } + + pub fn current_dir() -> Result { + let mut binding = crate::pipe::HOST.lock(); + let host = binding.get_mut().unwrap(); + match host.request(&control::Request::CurrentDir) { + control::Response::CurrentDir(path) => Ok(path), + control::Response::Err(error) => Err(error.into()), + _ => Err(control::ErrorKind::Other), + } + } } use super::pipe::HostPipe; diff --git a/vmm/src/comm.rs b/vmm/src/comm.rs index 61172a54..11868d54 100644 --- a/vmm/src/comm.rs +++ b/vmm/src/comm.rs @@ -27,6 +27,13 @@ pub fn control_thread(argv: Vec, mut pipe: ControlPipe) { loop { let response = match pipe.recv() { Request::GetArgs => Response::Args(argv.clone()), + Request::CurrentDir => match std::env::current_dir() { + Ok(path) => match path.into_os_string().into_string() { + Ok(path) => Response::CurrentDir(path), + Err(_) => Response::Err(IoErrorKind::InvalidData), + }, + Err(error) => Response::Err(error.kind().into()), + }, Request::Exit(code) => std::process::exit(code), Request::Open(path, mode) => { let f = OpenOptions::new()