diff --git a/crates/synapse-core/src/lib.rs b/crates/synapse-core/src/lib.rs index fa7ce5e..9b770b7 100644 --- a/crates/synapse-core/src/lib.rs +++ b/crates/synapse-core/src/lib.rs @@ -29,8 +29,9 @@ pub use revision::{ }; pub use revision_state::RevisionState; pub use tree::{ - tree_entry_order, validate_tree_entry_name, TreeDigest, TreeNodeEncoder, DOMAIN_TREE_V1, - MAX_TREE_ENTRY_NAME_BYTES, TREE_TAG_DIRECTORY, + decode_tree_node, tree_entry_order, validate_tree_entry_name, TreeDigest, TreeNodeEncoder, + TreeNodeEntry, TreeNodeEntryKind, DOMAIN_TREE_V1, MAX_TREE_ENTRY_NAME_BYTES, + TREE_TAG_DIRECTORY, }; /// Current canonical changeset envelope schema version. @@ -159,8 +160,8 @@ pub enum RefusalCode { /// without the mover's private key (ADR 0003 sections 5.3 and 5.4, step B4, issue #89). RevisionStateMoveUnverified, /// Strict recovery of a store opened with a revision seal key found a revision record - /// (kind 7, 8, 9, or 11) whose server seal is missing or does not verify against the seal - /// chain so far: it was written, changed, reordered, or followed by a removed record + /// (kind 7, 8, 9, 10, or 11) whose server seal is missing or does not verify against the + /// seal chain so far: it was written, changed, reordered, or followed by a removed record /// without the server's key (issue #101). RevisionRecordUnsealed, /// A `DerivationAttestation` does not name the revision it is presented with, or its @@ -193,6 +194,12 @@ pub enum RefusalCode { RadicleQuorumInsufficient, StaleBaseHead, SplitBrainDivergence, + /// A Git object the native model cannot hold without loss, so it could not be exported + /// with its original object ID (ADR 0003 step H1): a gitlink or empty subtree, an entry + /// name that is not a valid path segment, a mode other than the canonical `100644`, + /// `100755`, `120000` and `40000`, or a commit or tag whose links are not written in + /// Git's canonical layout. + GitObjectNotRepresentable, // Content admission refusals /// An admitted blob matches a rule of the versioned secret-scanning rule set (a private key @@ -375,6 +382,31 @@ impl fmt::Display for BlobDigest { } } +/// The Git tree payload a canonical tree node (C1, version 1) stands for: per entry, in the +/// node's order (which is Git's tree order), `{mode} {name}\0{oid}`, where a file's mode is +/// [`FileMode::git_octal_str`] and a directory's is `40000`. `blob_oid` and `tree_oid` give +/// the Git ID of each entry. Both the Git bridge's export and the store's check of a mapped +/// tree's Git ID use this one function. +pub fn git_tree_payload_of_node( + node: &[u8], + blob_oid: &mut dyn FnMut(BlobDigest) -> SynapseResult, + tree_oid: &mut dyn FnMut(TreeDigest) -> SynapseResult, +) -> SynapseResult> { + let mut out = Vec::with_capacity(node.len()); + for e in decode_tree_node(node)? { + let (mode, oid) = match e.kind { + TreeNodeEntryKind::File { mode, blob, .. } => (mode.git_octal_str(), blob_oid(blob)?), + TreeNodeEntryKind::Dir(child) => ("40000", tree_oid(child)?), + }; + out.extend_from_slice(mode.as_bytes()); + out.push(b' '); + out.extend_from_slice(e.name.as_bytes()); + out.push(0); + out.extend_from_slice(&oid.0); + } + Ok(out) +} + /// 20-byte standard Git SHA-1 object identifier (`blob`, `tree`, or `commit`). /// Distinct from `ChangesetId` and `BlobDigest`. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] diff --git a/crates/synapse-core/src/tree.rs b/crates/synapse-core/src/tree.rs index b4799c6..0274197 100644 --- a/crates/synapse-core/src/tree.rs +++ b/crates/synapse-core/src/tree.rs @@ -228,11 +228,119 @@ impl TreeNodeEncoder { } } +/// What one entry of a decoded tree node names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TreeNodeEntryKind { + /// A file: its mode, `BlobDigest`, and size in bytes. + File { + mode: FileMode, + blob: crate::BlobDigest, + size_bytes: u64, + }, + /// A subdirectory: the `TreeDigest` of its node. + Dir(TreeDigest), +} + +/// One entry of a decoded tree node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TreeNodeEntry { + pub name: String, + pub kind: TreeNodeEntryKind, +} + +/// Smallest encoded entry: tag, a one-byte name and a directory digest. +const MIN_TREE_ENTRY_BYTES: usize = 1 + 2 + 1 + 32; + +/// Decode exactly one canonical tree node (version 1). The count is checked against the bytes +/// that remain before anything is reserved; the node is accepted only if re-encoding its +/// entries through [`TreeNodeEncoder`] gives back exactly `bytes`, so names, order, duplicates +/// and tags follow the encoder's rules. Any other input is `TreeNodeMalformed`. +pub fn decode_tree_node(bytes: &[u8]) -> SynapseResult> { + let bad = |msg: &str| SynapseError::new(RefusalCode::TreeNodeMalformed, msg.to_string()); + let take = |pos: &mut usize, n: usize| -> SynapseResult<&[u8]> { + let end = pos + .checked_add(n) + .filter(|&e| e <= bytes.len()) + .ok_or_else(|| bad("tree node truncated"))?; + let s = &bytes[*pos..end]; + *pos = end; + Ok(s) + }; + let mut pos = 0usize; + let count = u32::from_le_bytes(take(&mut pos, 4)?.try_into().expect("4 bytes")) as usize; + if count.saturating_mul(MIN_TREE_ENTRY_BYTES) > bytes.len() - pos { + return Err(bad( + "tree node declares more entries than its bytes can hold", + )); + } + let mut entries = Vec::with_capacity(count); + let mut enc = TreeNodeEncoder::new(count as u32); + for _ in 0..count { + let tag = take(&mut pos, 1)?[0]; + let len = u16::from_le_bytes(take(&mut pos, 2)?.try_into().expect("2 bytes")) as usize; + let name = std::str::from_utf8(take(&mut pos, len)?) + .map_err(|_| bad("tree entry name is not UTF-8"))? + .to_string(); + let kind = if tag == TREE_TAG_DIRECTORY { + let d = TreeDigest(take(&mut pos, 32)?.try_into().expect("32 bytes")); + enc.dir(&name, &d)?; + TreeNodeEntryKind::Dir(d) + } else { + let mode = FileMode::from_tag_byte(tag).ok_or_else(|| bad("unknown tree entry tag"))?; + let blob = crate::BlobDigest(take(&mut pos, 32)?.try_into().expect("32 bytes")); + let size_bytes = u64::from_le_bytes(take(&mut pos, 8)?.try_into().expect("8 bytes")); + enc.file(&name, mode, &blob, size_bytes)?; + TreeNodeEntryKind::File { + mode, + blob, + size_bytes, + } + }; + entries.push(TreeNodeEntry { name, kind }); + } + if pos != bytes.len() { + return Err(bad("tree node has trailing bytes")); + } + let (_, again) = enc.finish()?; + if again != bytes { + return Err(bad("tree node is not in its canonical encoding")); + } + Ok(entries) +} + #[cfg(test)] mod tests { use super::*; use crate::BlobDigest; + #[test] + fn h1_tree_node_decodes_canonically_and_refuses_every_other_form() { + let blob = BlobDigest::compute(b"x"); + let mut enc = TreeNodeEncoder::new(2); + enc.file("a", FileMode::Executable755, &blob, 1).unwrap(); + enc.dir("b", &TreeDigest::empty()).unwrap(); + let (_, bytes) = enc.finish().unwrap(); + let entries = decode_tree_node(&bytes).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[1].kind, TreeNodeEntryKind::Dir(TreeDigest::empty())); + assert_eq!(decode_tree_node(&0u32.to_le_bytes()).unwrap(), vec![]); + for cut in 0..bytes.len() { + assert!( + decode_tree_node(&bytes[..cut]).is_err(), + "truncated at {cut}" + ); + } + let mut extra = bytes.clone(); + extra.push(0); + assert!(decode_tree_node(&extra).is_err(), "trailing byte"); + let mut huge = bytes.clone(); + huge[..4].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(decode_tree_node(&huge).is_err(), "count over the input"); + let mut tag = bytes.clone(); + tag[4] = 9; + assert!(decode_tree_node(&tag).is_err(), "unknown tag"); + } + #[test] fn empty_tree_digest_is_the_domain_separated_hash_of_a_zero_count() { let mut h = Sha256::new(); diff --git a/crates/synapse-git/src/bridge.rs b/crates/synapse-git/src/bridge.rs new file mode 100644 index 0000000..fad2814 --- /dev/null +++ b/crates/synapse-git/src/bridge.rs @@ -0,0 +1,890 @@ +//! ADR 0003 step H1 cut 2: the native side of the Git bridge (section 5.10). See +//! `docs/H1_GIT_BRIDGE.md`. +//! +//! * [`import_repository`] / [`import_history`]: a repository's full history into the store as +//! WAL kind 10 records (`synapse_store::git_bridge`). Blobs become stored blobs, trees become +//! canonical `TreeDigest` nodes (C1), commits become imported revisions that keep every byte +//! after their `tree` and `parent` lines, annotated tags keep every byte after their `object` +//! line, and the references are recorded last. Every object is converted and checked before +//! the first record is written, so a refused repository leaves the store unchanged. +//! * [`export_store`]: a new bare repository from the store alone. Imported objects are rebuilt +//! from their native form and the stored bytes, and each must hash to the Git ID it was +//! imported under. Native revisions (envelope v2) are written with generated, deterministic +//! commit fields. + +use crate::object::{parse_commit_links, parse_tag_target, parse_tree, GitTreeEntry}; +use crate::repo::{export_repository, ExportReceipt, GitHistory, GitRepository, ReadLimits}; +use crate::{GitObject, ObjectKind}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::Path; +use synapse_core::{ + decode_tree_node, git_tree_payload_of_node, tree_entry_order, validate_tree_entry_name, + BlobDigest, ChangesetId, FileMode, GitOid, MutationOp, RefusalCode, RepoId, SynapseError, + SynapseResult, TreeDigest, TreeNodeEncoder, TreeNodeEntryKind, +}; +use synapse_store::git_bridge::{ + check_git_ref_names, GitBlobMapping, GitBridgeEntry, GitBridgeRepo, GitBridgeTarget, + GitImportRecord, GitImportedCommit, GitImportedRefs, GitImportedTag, GitTreeMapping, +}; +use synapse_store::SynapseStore; + +/// Most object mappings in one `Objects` record. +pub const MAX_MAPPINGS_PER_RECORD: usize = 4096; + +/// Author email of an exported native revision (its name is the first author's DID). +pub const NATIVE_AUTHOR_EMAIL: &str = "agent@synapse.aienos.local"; + +/// Committer line identity of an exported native revision. +pub const NATIVE_COMMITTER: &str = "Synapse Engine "; + +fn not_representable(msg: impl Into) -> SynapseError { + SynapseError::new(RefusalCode::GitObjectNotRepresentable, msg) +} + +fn corrupt(msg: impl Into) -> SynapseError { + SynapseError::new(RefusalCode::GitObjectCorrupt, msg) +} + +/// What an import wrote. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ImportReceipt { + pub blobs: usize, + pub trees: usize, + pub commits: usize, + pub tags: usize, + pub refs: usize, + /// Kind 10 records written. + pub records_written: usize, + /// Records skipped because the store already held them (a repeated import). + pub records_skipped: usize, +} + +/// Read `git_dir`'s full history under `limits` and import it as `repo_id`. +pub fn import_repository( + store: &mut SynapseStore, + repo_id: &RepoId, + git_dir: &Path, + limits: ReadLimits, +) -> SynapseResult { + let history = GitRepository::open_with_limits(git_dir, limits)?.full_history()?; + import_history(store, repo_id, &history) +} + +/// A Git tree entry mode as a native entry: a file mode, or `None` for a subtree. Only the +/// modes Git writes today are representable (`100644`, `100755`, `120000`, `40000`). +fn native_mode(mode: &[u8]) -> SynapseResult> { + match mode { + b"100644" => Ok(Some(FileMode::Regular644)), + b"100755" => Ok(Some(FileMode::Executable755)), + b"120000" => Ok(Some(FileMode::Symlink120000)), + b"40000" => Ok(None), + other => Err(not_representable(format!( + "tree entry mode {:?} has no native form (gitlinks and legacy modes are not imported)", + String::from_utf8_lossy(other) + ))), + } +} + +/// The Git tree a canonical node stands for. Entries keep the node's order, which is Git's +/// tree order; `blob_oid` and `tree_oid` give the Git ID of each entry. +fn git_tree_of_node( + node: &[u8], + blob_oid: &dyn Fn(BlobDigest) -> SynapseResult, + tree_oid: &dyn Fn(TreeDigest) -> SynapseResult, +) -> SynapseResult { + let payload = git_tree_payload_of_node(node, &mut |b| blob_oid(b), &mut |t| tree_oid(t))?; + Ok(GitObject::new(ObjectKind::Tree, payload)) +} + +/// `tree \n` then `parent \n` for each parent: the part of a commit the native +/// form stores as links. +fn commit_link_lines(tree: GitOid, parents: &[GitOid]) -> Vec { + let mut out = format!("tree {}\n", tree.to_hex()).into_bytes(); + for p in parents { + out.extend_from_slice(format!("parent {}\n", p.to_hex()).as_bytes()); + } + out +} + +/// Order `oids` so every object comes after the objects `links` names (iterative depth first, +/// so a long history cannot overflow the stack). A link to an object outside `oids` is +/// ignored here (the caller resolves it). +fn links_first( + oids: &[GitOid], + links: &dyn Fn(GitOid) -> SynapseResult>, +) -> SynapseResult> { + let members: std::collections::HashSet = oids.iter().copied().collect(); + let mut done: std::collections::HashSet = std::collections::HashSet::new(); + let mut order = Vec::with_capacity(oids.len()); + for &root in oids { + let mut stack = vec![(root, false)]; + while let Some((oid, expanded)) = stack.pop() { + if done.contains(&oid) { + continue; + } + if expanded { + done.insert(oid); + order.push(oid); + continue; + } + stack.push((oid, true)); + for l in links(oid)? { + if members.contains(&l) && !done.contains(&l) { + stack.push((l, false)); + } + } + } + } + Ok(order) +} + +/// Import `history` as `repo_id`. Every object is converted first; a refusal (unrepresentable +/// tree, non-canonical commit or tag layout, invalid reference name, a blob over the store's +/// blob cap) happens before anything is written. Records then go in dependency order: objects +/// (blobs, then trees, children first), commits (parents first), tags, and one `Refs` record. +/// Importing the same history again writes nothing. +pub fn import_history( + store: &mut SynapseStore, + repo_id: &RepoId, + history: &GitHistory, +) -> SynapseResult { + let limits = store.limits().clone(); + let of_kind = |k: ObjectKind| -> Vec { + history + .objects + .iter() + .filter(|(_, o)| o.kind == k) + .map(|(oid, _)| *oid) + .collect() + }; + let object = |oid: GitOid| -> SynapseResult<&GitObject> { + history + .objects + .get(&oid) + .ok_or_else(|| corrupt(format!("object {oid} is not in the history"))) + }; + + // Blobs. + let mut blobs: HashMap = HashMap::new(); + let mut blob_oids: HashMap = HashMap::new(); + let blob_list = of_kind(ObjectKind::Blob); + for &oid in &blob_list { + let payload = &object(oid)?.payload; + let size = payload.len() as u64; + if size > limits.max_blob_bytes { + return Err(SynapseError::new( + RefusalCode::QuotaPayloadTooLarge, + format!( + "Git blob {oid} holds {size} bytes, over the store's max_blob_bytes {}", + limits.max_blob_bytes + ), + )); + } + let digest = BlobDigest::compute(payload); + blobs.insert(oid, (digest, size)); + blob_oids.insert(digest, oid); + } + + // Trees, children first. + let tree_list = of_kind(ObjectKind::Tree); + let tree_order = links_first(&tree_list, &|oid| { + Ok(parse_tree(&object(oid)?.payload)? + .into_iter() + .filter(|e| e.mode == b"40000" || e.mode == b"040000") + .map(|e| e.oid) + .collect()) + })?; + let mut trees: HashMap)> = HashMap::new(); + let mut tree_oids: HashMap = HashMap::new(); + for &oid in &tree_order { + let entries = parse_tree(&object(oid)?.payload)?; + let count = u32::try_from(entries.len()) + .map_err(|_| not_representable(format!("tree {oid} has too many entries")))?; + let mut enc = TreeNodeEncoder::new(count); + for e in &entries { + let name = std::str::from_utf8(&e.name).map_err(|_| { + not_representable(format!("tree {oid} has an entry name that is not UTF-8")) + })?; + validate_tree_entry_name(name) + .map_err(|err| not_representable(format!("tree {oid}: {}", err.message)))?; + let pushed = match native_mode(&e.mode)? { + Some(mode) => { + let (blob, size) = *blobs.get(&e.oid).ok_or_else(|| { + corrupt(format!("tree {oid} names {} which is not a blob", e.oid)) + })?; + enc.file(name, mode, &blob, size) + } + None => { + let (child, _) = trees.get(&e.oid).ok_or_else(|| { + corrupt(format!("tree {oid} names {} which is not a tree", e.oid)) + })?; + if *child == TreeDigest::empty() { + return Err(not_representable(format!( + "tree {oid} holds an empty subtree {name:?}" + ))); + } + enc.dir(name, child) + } + }; + pushed.map_err(|err| not_representable(format!("tree {oid}: {}", err.message)))?; + } + let (digest, node) = enc + .finish() + .map_err(|err| not_representable(format!("tree {oid}: {}", err.message)))?; + // The node must give back exactly this tree (mode spelling, order, names). + let rebuilt = git_tree_of_node( + &node, + &|b| { + blob_oids + .get(&b) + .copied() + .ok_or_else(|| corrupt(format!("blob {b} has no Git ID"))) + }, + &|t| { + tree_oids + .get(&t) + .copied() + .ok_or_else(|| corrupt(format!("tree {t} has no Git ID"))) + }, + )?; + if rebuilt.oid() != oid { + return Err(not_representable(format!( + "tree {oid} is not in Git's canonical layout (its native form rebuilds as {})", + rebuilt.oid() + ))); + } + tree_oids.insert(digest, oid); + trees.insert(oid, (digest, node)); + } + + // Commits, parents first. + let commit_list = of_kind(ObjectKind::Commit); + let commit_order = links_first(&commit_list, &|oid| { + Ok(parse_commit_links(&object(oid)?.payload)?.parents) + })?; + let mut commits: HashMap = HashMap::new(); + let mut commit_entries = Vec::with_capacity(commit_order.len()); + for &oid in &commit_order { + let payload = &object(oid)?.payload; + let links = parse_commit_links(payload)?; + let prefix = commit_link_lines(links.tree, &links.parents); + if !payload.starts_with(&prefix) { + return Err(not_representable(format!( + "commit {oid} does not start with its tree and parent lines in canonical form" + ))); + } + let (tree, _) = trees + .get(&links.tree) + .ok_or_else(|| corrupt(format!("commit {oid} names tree {} not read", links.tree)))?; + let parents = links + .parents + .iter() + .map(|p| { + commits + .get(p) + .copied() + .ok_or_else(|| corrupt(format!("commit {oid} names parent {p} not read"))) + }) + .collect::>>()?; + let c = GitImportedCommit { + git_oid: oid, + tree: *tree, + parents, + headers: payload[prefix.len()..].to_vec(), + }; + commits.insert(oid, c.changeset_id()); + commit_entries.push(c); + } + + // Annotated tags, inner tags first. + let target_of = |oid: GitOid, tags: &HashMap| -> SynapseResult { + let kind = object(oid)?.kind; + let missing = || corrupt(format!("{} {oid} was not imported", kind.name())); + Ok(match kind { + ObjectKind::Commit => GitBridgeTarget::Commit(*commits.get(&oid).ok_or_else(missing)?), + ObjectKind::Tree => GitBridgeTarget::Tree(trees.get(&oid).ok_or_else(missing)?.0), + ObjectKind::Blob => GitBridgeTarget::Blob(blobs.get(&oid).ok_or_else(missing)?.0), + ObjectKind::Tag => { + tags.get(&oid).ok_or_else(missing)?; + GitBridgeTarget::Tag(oid) + } + }) + }; + let tag_list = of_kind(ObjectKind::Tag); + let tag_order = links_first(&tag_list, &|oid| { + Ok(vec![parse_tag_target(&object(oid)?.payload)?.0]) + })?; + let mut tags: HashMap = HashMap::new(); + let mut tag_entries = Vec::with_capacity(tag_order.len()); + for &oid in &tag_order { + let payload = &object(oid)?.payload; + let (target, _) = parse_tag_target(payload)?; + let prefix = format!("object {}\n", target.to_hex()); + if !payload.starts_with(prefix.as_bytes()) { + return Err(not_representable(format!( + "tag {oid} does not start with its object line in canonical form" + ))); + } + tag_entries.push(GitImportedTag { + git_oid: oid, + target: target_of(target, &tags)?, + rest: payload[prefix.len()..].to_vec(), + }); + tags.insert(oid, ()); + } + + // References (checked now, written last). + check_git_ref_names( + history.refs.keys().map(String::as_str), + history.head.as_deref(), + )?; + let refs = GitImportedRefs { + refs: history + .refs + .iter() + .map(|(name, oid)| Ok((name.clone(), target_of(*oid, &tags)?))) + .collect::>>()?, + head: history.head.clone(), + }; + + // Build every record, then hand them to the store as one batch: it checks them all + // (limits, frame size, every Git ID) before the first write, so a refusal writes nothing. + let mut receipt = ImportReceipt { + blobs: blob_list.len(), + trees: tree_order.len(), + commits: commit_entries.len(), + tags: tag_entries.len(), + refs: refs.refs.len(), + ..ImportReceipt::default() + }; + let mut records: Vec = Vec::new(); + // Objects records: at most MAX_MAPPINGS_PER_RECORD mappings, inline blob bytes within + // max_total_changeset_bytes, and an upper bound of the WAL frame (metadata per mapping + // plus inline blobs) within max_wal_record_bytes. A single object over either limit + // goes alone into a record the store then refuses. + let budget = limits.max_total_changeset_bytes; + let frame_cap = limits.max_wal_record_bytes; + let frame_base = 64 + repo_id.as_str().len() as u64; + let mut batch_blobs: Vec = Vec::new(); + let mut batch_bytes: Vec<(BlobDigest, &[u8])> = Vec::new(); + let mut batch_trees: Vec = Vec::new(); + let mut used = 0u64; + let mut frame = frame_base; + #[allow(clippy::too_many_arguments)] + fn flush<'a>( + records: &mut Vec>, + batch_blobs: &mut Vec, + batch_bytes: &mut Vec<(BlobDigest, &'a [u8])>, + batch_trees: &mut Vec, + used: &mut u64, + frame: &mut u64, + frame_base: u64, + ) { + if batch_blobs.is_empty() && batch_trees.is_empty() { + return; + } + let entry = GitBridgeEntry::Objects { + blobs: std::mem::take(batch_blobs), + trees: std::mem::take(batch_trees), + }; + records.push((entry, std::mem::take(batch_bytes))); + *used = 0; + *frame = frame_base; + } + for &oid in &blob_list { + let payload = object(oid)?.payload.as_slice(); + let (digest, size) = blobs[&oid]; + // Mapping (20 + 32 + 8) plus inline blob (32 + 4 + bytes). + let grows = 96 + size; + if batch_blobs.len() == MAX_MAPPINGS_PER_RECORD + || used + size > budget + || frame + grows > frame_cap + { + flush( + &mut records, + &mut batch_blobs, + &mut batch_bytes, + &mut batch_trees, + &mut used, + &mut frame, + frame_base, + ); + } + batch_blobs.push(GitBlobMapping { + git_oid: oid, + blob: digest, + size_bytes: size, + }); + batch_bytes.push((digest, payload)); + used += size; + frame += grows; + } + for &oid in &tree_order { + let (_, node) = &trees[&oid]; + let size = node.len() as u64; + // Git ID (20) plus length (4) plus node. + let grows = 24 + size; + if batch_blobs.len() + batch_trees.len() >= MAX_MAPPINGS_PER_RECORD + || used + size > budget + || frame + grows > frame_cap + { + flush( + &mut records, + &mut batch_blobs, + &mut batch_bytes, + &mut batch_trees, + &mut used, + &mut frame, + frame_base, + ); + } + batch_trees.push(GitTreeMapping { + git_oid: oid, + node: node.clone(), + }); + used += size; + frame += grows; + } + flush( + &mut records, + &mut batch_blobs, + &mut batch_bytes, + &mut batch_trees, + &mut used, + &mut frame, + frame_base, + ); + for c in commit_entries { + records.push((GitBridgeEntry::Commit(c), Vec::new())); + } + for t in tag_entries { + records.push((GitBridgeEntry::Tag(t), Vec::new())); + } + records.push((GitBridgeEntry::Refs(refs), Vec::new())); + for seq in store.record_git_imports(repo_id, records)? { + match seq { + Some(_) => receipt.records_written += 1, + None => receipt.records_skipped += 1, + } + } + Ok(receipt) +} + +/// Git IDs of the imported objects of one repository, by native identity. +struct ImportedIds { + blobs: HashMap, + trees: HashMap, + commits: HashMap, + tags: HashSet, +} + +impl ImportedIds { + fn of(repo: Option<&GitBridgeRepo>) -> Self { + let mut ids = Self { + blobs: HashMap::new(), + trees: HashMap::new(), + commits: HashMap::new(), + tags: HashSet::new(), + }; + if let Some(r) = repo { + ids.blobs = r.blobs().iter().map(|(oid, (d, _))| (*d, *oid)).collect(); + ids.trees = r.trees().iter().map(|(oid, d)| (*d, *oid)).collect(); + ids.commits = r.commits().map(|(oid, (id, _))| (*id, *oid)).collect(); + ids.tags = r.tags().keys().copied().collect(); + } + ids + } + + fn target(&self, t: &GitBridgeTarget) -> SynapseResult { + let missing = || corrupt(format!("bridge target {t:?} has no Git ID")); + Ok(match t { + GitBridgeTarget::Commit(id) => *self.commits.get(id).ok_or_else(missing)?, + GitBridgeTarget::Tree(d) => *self.trees.get(d).ok_or_else(missing)?, + GitBridgeTarget::Blob(b) => *self.blobs.get(b).ok_or_else(missing)?, + GitBridgeTarget::Tag(oid) => *self.tags.get(oid).ok_or_else(missing)?, + }) + } +} + +/// One file of a flattened native tree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FlatFile { + mode: FileMode, + blob: BlobDigest, + size: u64, +} + +/// Export state: every object written so far, by Git ID. +struct Exporter<'a> { + store: &'a SynapseStore, + repo_id: &'a RepoId, + objects: BTreeMap, + /// Git blob IDs of blobs read for native trees. + native_blobs: HashMap, + /// Result trees of the native revisions exported so far (a native child names its + /// parent's result as its `base_tree`). + native_trees: HashMap>, + ids: ImportedIds, +} + +impl Exporter<'_> { + fn add(&mut self, object: GitObject) -> GitOid { + let oid = object.oid(); + self.objects.entry(oid).or_insert(object); + oid + } + + /// Add `object`, which must hash to the Git ID it was imported under. + fn add_checked(&mut self, object: GitObject, expected: GitOid) -> SynapseResult<()> { + let oid = object.oid(); + if oid != expected { + return Err(corrupt(format!( + "stored {} rebuilds as {oid}, not its imported Git ID {expected}", + object.kind.name() + ))); + } + self.objects.entry(oid).or_insert(object); + Ok(()) + } + + fn blob_oid(&mut self, blob: BlobDigest) -> SynapseResult { + if let Some(oid) = self.ids.blobs.get(&blob).or(self.native_blobs.get(&blob)) { + return Ok(*oid); + } + let bytes = self.store.get_blob_visible(blob, None, None)?; + let oid = self.add(GitObject::new(ObjectKind::Blob, bytes)); + self.native_blobs.insert(blob, oid); + Ok(oid) + } + + fn export_imported(&mut self, repo: &GitBridgeRepo) -> SynapseResult<()> { + for (oid, (digest, size)) in repo.blobs() { + let bytes = self.store.get_blob_visible(*digest, None, None)?; + if bytes.len() as u64 != *size { + return Err(corrupt(format!( + "stored blob of Git blob {oid} holds {} bytes, not {size}", + bytes.len() + ))); + } + self.add_checked(GitObject::new(ObjectKind::Blob, bytes), *oid)?; + } + for (oid, digest) in repo.trees() { + let node = self.store.git_bridge_tree_node(*digest).ok_or_else(|| { + SynapseError::new( + RefusalCode::TreeNodeNotFound, + format!("tree node {digest} of Git tree {oid} is not stored"), + ) + })?; + let ids = &self.ids; + let tree = git_tree_of_node(node, &|b| ids.target(&GitBridgeTarget::Blob(b)), &|t| { + ids.target(&GitBridgeTarget::Tree(t)) + })?; + self.add_checked(tree, *oid)?; + } + for (oid, (_, c)) in repo.commits() { + let tree = self.ids.target(&GitBridgeTarget::Tree(c.tree))?; + let parents = c + .parents + .iter() + .map(|p| self.ids.target(&GitBridgeTarget::Commit(*p))) + .collect::>>()?; + let mut payload = commit_link_lines(tree, &parents); + payload.extend_from_slice(&c.headers); + self.add_checked(GitObject::new(ObjectKind::Commit, payload), *oid)?; + } + for (oid, t) in repo.tags() { + let target = self.ids.target(&t.target)?; + let mut payload = format!("object {}\n", target.to_hex()).into_bytes(); + payload.extend_from_slice(&t.rest); + self.add_checked(GitObject::new(ObjectKind::Tag, payload), *oid)?; + } + Ok(()) + } + + /// Flatten the tree `root` (an imported tree, a native revision's result tree exported + /// before, or the empty tree) into path to file. + fn flatten(&self, root: TreeDigest) -> SynapseResult> { + if let Some(files) = self.native_trees.get(&root) { + return Ok(files.clone()); + } + let mut files = BTreeMap::new(); + let mut stack = vec![(String::new(), root)]; + while let Some((prefix, digest)) = stack.pop() { + if digest == TreeDigest::empty() { + continue; + } + let node = self.store.git_bridge_tree_node(digest).ok_or_else(|| { + SynapseError::new( + RefusalCode::TreeNodeNotFound, + format!("base tree node {digest} is not stored in the bridge"), + ) + })?; + for e in decode_tree_node(node)? { + let path = format!("{prefix}{}", e.name); + match e.kind { + TreeNodeEntryKind::File { + mode, + blob, + size_bytes, + } => { + files.insert( + path, + FlatFile { + mode, + blob, + size: size_bytes, + }, + ); + } + TreeNodeEntryKind::Dir(child) => stack.push((format!("{path}/"), child)), + } + } + } + Ok(files) + } + + /// Write the Git trees of `files` (a flat path map); returns the root tree's Git ID and + /// its C1 `TreeDigest` (the same canonical nodes the tree store builds). + fn write_tree( + &mut self, + files: &BTreeMap, + ) -> SynapseResult<(GitOid, TreeDigest)> { + enum Node { + File(FlatFile), + Dir(BTreeMap), + } + enum Node2 { + File(FlatFile), + Dir(TreeDigest), + } + let mut root: BTreeMap = BTreeMap::new(); + for (path, f) in files { + let mut dir = &mut root; + let mut segs: Vec<&str> = path.split('/').collect(); + let name = segs.pop().unwrap_or_default(); + for s in segs { + let slot = dir + .entry(s.to_string()) + .or_insert_with(|| Node::Dir(BTreeMap::new())); + dir = match slot { + Node::Dir(d) => d, + Node::File(_) => { + return Err(corrupt(format!("path {path} lies below a file"))); + } + }; + } + dir.insert(name.to_string(), Node::File(*f)); + } + fn emit( + ex: &mut Exporter<'_>, + dir: &BTreeMap, + ) -> SynapseResult<(GitOid, TreeDigest)> { + let mut entries: Vec<(bool, &str, GitTreeEntry, Node2)> = Vec::new(); + for (name, node) in dir { + let (is_dir, mode, oid, native) = match node { + Node::File(f) => ( + false, + f.mode.git_octal_str().as_bytes().to_vec(), + ex.blob_oid(f.blob)?, + Node2::File(*f), + ), + Node::Dir(d) => { + let (oid, digest) = emit(ex, d)?; + (true, b"40000".to_vec(), oid, Node2::Dir(digest)) + } + }; + entries.push(( + is_dir, + name, + GitTreeEntry { + mode, + name: name.as_bytes().to_vec(), + oid, + }, + native, + )); + } + entries.sort_by(|a, b| tree_entry_order(a.1, a.0, b.1, b.0)); + let count = u32::try_from(entries.len()) + .map_err(|_| not_representable("a native tree has too many entries"))?; + let mut enc = TreeNodeEncoder::new(count); + for (_, name, _, native) in &entries { + match native { + Node2::File(f) => enc.file(name, f.mode, &f.blob, f.size)?, + Node2::Dir(d) => enc.dir(name, d)?, + } + } + let (digest, _) = enc.finish()?; + let list: Vec = entries.into_iter().map(|e| e.2).collect(); + let oid = ex.add(GitObject::new( + ObjectKind::Tree, + crate::object::encode_tree(&list), + )); + Ok((oid, digest)) + } + emit(self, &root) + } + + /// The Git commit of native revision `id` (and of its native ancestors), deterministic: + /// tree = `base_tree` (the empty tree, an imported tree, or the result tree of a native + /// revision exported before it) with the revision's mutations applied (an upsert replaces a file at + /// an ancestor path and everything below its own path; a delete removes only a file); + /// parents in the revision's order (imported commits by their Git ID); author the first + /// attestation's DID with `NATIVE_AUTHOR_EMAIL`, committer `NATIVE_COMMITTER`, both at + /// time 0 +0000; message the intent summary, a blank line, and `Synapse-Changeset-Id` and + /// `Synapse-Change-Id` trailers. + fn native_commit( + &mut self, + id: ChangesetId, + done: &mut HashMap, + ) -> SynapseResult { + let mut stack = vec![(id, false)]; + while let Some((cs, expanded)) = stack.pop() { + if done.contains_key(&cs) || self.ids.commits.contains_key(&cs) { + continue; + } + let rev = self.store.get_revision(self.repo_id, cs)?; + if !expanded { + stack.push((cs, true)); + for p in &rev.content.parents { + if !done.contains_key(p) && !self.ids.commits.contains_key(p) { + stack.push((*p, false)); + } + } + continue; + } + let mut files = self.flatten(rev.content.base_tree)?; + for m in &rev.content.mutations { + match &m.op { + MutationOp::Upsert { + mode, + blob_digest, + size_bytes, + } => { + for (i, _) in m.path.match_indices('/') { + files.remove(&m.path[..i]); + } + let below = format!("{}/", m.path); + let doomed: Vec = files + .range(below.clone()..) + .take_while(|(p, _)| p.starts_with(&below)) + .map(|(p, _)| p.clone()) + .collect(); + for p in doomed { + files.remove(&p); + } + files.insert( + m.path.clone(), + FlatFile { + mode: *mode, + blob: *blob_digest, + size: *size_bytes, + }, + ); + } + MutationOp::Delete => { + files.remove(&m.path); + } + } + } + let (tree, digest) = self.write_tree(&files)?; + self.native_trees.insert(digest, files); + let parents = rev + .content + .parents + .iter() + .map(|p| { + done.get(p) + .or(self.ids.commits.get(p)) + .copied() + .ok_or_else(|| corrupt(format!("parent {p} has no Git commit"))) + }) + .collect::>>()?; + let author = rev + .attestations + .first() + .map(|a| a.author_did()) + .ok_or_else(|| corrupt(format!("revision {cs} has no attestation")))?; + let mut payload = commit_link_lines(tree, &parents); + payload.extend_from_slice( + format!( + "author {author} <{NATIVE_AUTHOR_EMAIL}> 0 +0000\ncommitter {NATIVE_COMMITTER} 0 +0000\n\n{}\n\nSynapse-Changeset-Id: {}\nSynapse-Change-Id: {}\n", + rev.content.intent_summary, + cs.to_hex(), + rev.content.change_id.to_hex() + ) + .as_bytes(), + ); + let oid = self.add(GitObject::new(ObjectKind::Commit, payload)); + done.insert(cs, oid); + } + done.get(&id) + .or(self.ids.commits.get(&id)) + .copied() + .ok_or_else(|| corrupt(format!("revision {id} was not exported"))) + } +} + +/// Export `repo_id` from `store` alone as the new bare repository `dest` (one pack). Imported +/// objects and references are rebuilt from the bridge records and stored blobs; each object +/// must hash to its imported Git ID (`GitObjectCorrupt` otherwise). `native_refs` adds a +/// reference for each native revision named (written with its native ancestors, see the +/// native commit rule in `docs/H1_GIT_BRIDGE.md`); a name already imported is `PathInvalid`. +/// `HEAD` is the imported `HEAD` (default `refs/heads/main`). The same store state always gives +/// the same pack bytes. +pub fn export_store( + store: &SynapseStore, + repo_id: &RepoId, + native_refs: &BTreeMap, + dest: &Path, +) -> SynapseResult { + let repo = store.git_bridge_repo(repo_id); + let mut ex = Exporter { + store, + repo_id, + objects: BTreeMap::new(), + native_blobs: HashMap::new(), + native_trees: HashMap::new(), + ids: ImportedIds::of(repo), + }; + let mut refs: BTreeMap = BTreeMap::new(); + let mut head = None; + if let Some(r) = repo { + ex.export_imported(r)?; + if let Some(imported) = r.refs() { + for (name, target) in &imported.refs { + refs.insert(name.clone(), ex.ids.target(target)?); + } + head = imported.head.clone(); + } + } + let mut done = HashMap::new(); + for (name, id) in native_refs { + if refs.contains_key(name) { + return Err(SynapseError::new( + RefusalCode::PathInvalid, + format!("native reference {name} is already an imported reference"), + )); + } + let oid = ex.native_commit(*id, &mut done)?; + refs.insert(name.clone(), oid); + } + check_git_ref_names(refs.keys().map(String::as_str), head.as_deref())?; + let objects: Vec = ex.objects.into_values().collect(); + export_repository(dest, &objects, &refs, head.as_deref()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Export resolves a reference or tag that names a tag only when that tag is imported + /// (review of #96: it used to pass any tag Git ID through). + #[test] + fn export_refuses_a_tag_target_that_is_not_imported() { + let ids = ImportedIds::of(None); + let oid = GitOid::compute_git_object("tag", b"never imported"); + let err = ids.target(&GitBridgeTarget::Tag(oid)).unwrap_err(); + assert_eq!(err.code, RefusalCode::GitObjectCorrupt, "{err}"); + } +} diff --git a/crates/synapse-git/src/lib.rs b/crates/synapse-git/src/lib.rs index cc97f10..ee8899b 100644 --- a/crates/synapse-git/src/lib.rs +++ b/crates/synapse-git/src/lib.rs @@ -8,11 +8,13 @@ //! and indexes, delta resolution), and [`repo`] (reading a repository's full history and //! exporting an object set as a new bare repository with one pack). +pub mod bridge; pub mod inflate; pub mod object; pub mod pack; pub mod repo; +pub use bridge::{export_store, import_history, import_repository, ImportReceipt}; pub use object::{GitObject, GitTreeEntry, ObjectKind}; pub use pack::{PackIndexEntry, PackReader, WrittenPack}; pub use repo::{export_repository, ExportReceipt, GitHistory, GitRepository, ReadLimits}; diff --git a/crates/synapse-git/src/pack.rs b/crates/synapse-git/src/pack.rs index 90fa070..b4c9707 100644 --- a/crates/synapse-git/src/pack.rs +++ b/crates/synapse-git/src/pack.rs @@ -352,15 +352,40 @@ impl PackReader { /// the trailing checksum. Every `OBJ_REF_DELTA` base must be in the pack (a self-contained /// pack, as `gitformat-pack` requires on disk). pub fn index(data: Vec) -> SynapseResult { + Self::index_with_limits(data, &crate::repo::ReadLimits::default()) + } + + /// [`Self::index`] under `limits`: no entry may declare more than `max_object_bytes`, and + /// the sizes every entry declares it resolves to (whole sizes, and delta result sizes read + /// from the first bytes of each delta stream) must fit `max_total_object_bytes` before any + /// entry is resolved, as when a repository is opened. Each entry is resolved once, plus + /// once more for each reference delta base it waits on (no repeated rounds over the pack). + pub fn index_with_limits( + data: Vec, + limits: &crate::repo::ReadLimits, + ) -> SynapseResult { + let max = limits.max_object_bytes; let count = check_pack_framing(&data)?; let body_end = data.len() - 20; - let mut offsets = Vec::with_capacity(count as usize); + let mut offsets = Vec::with_capacity((count as usize).min(body_end / 2)); let mut pos = 12usize; + let mut declared = 0u64; for _ in 0..count { if pos >= body_end { return Err(corrupt("pack holds fewer entries than its header declares")); } - let (_, next) = read_raw_entry(&data[..body_end], pos as u64, MAX_PACK_OBJECT_BYTES)?; + let size = declared_object_size(&data[..body_end], pos as u64, max)?; + declared = declared.saturating_add(size); + if declared > limits.max_total_object_bytes { + return Err(SynapseError::new( + RefusalCode::QuotaPayloadTooLarge, + format!( + "pack entries declare over the {} byte budget", + limits.max_total_object_bytes + ), + )); + } + let (_, next) = read_raw_entry(&data[..body_end], pos as u64, max)?; offsets.push((pos as u64, crc32(&data[pos..next]))); pos = next; } @@ -375,38 +400,36 @@ impl PackReader { entries: Vec::with_capacity(offsets.len()), cache: HashMap::new(), cache_bytes: 0, - max_object_bytes: MAX_PACK_OBJECT_BYTES, + max_object_bytes: max, }; - // Whole objects and offset deltas resolve by position alone; reference deltas need - // their base's ID first, so repeat until every entry is resolved or nothing moves. - let mut pending: Vec<(u64, u32)> = offsets; - while !pending.is_empty() { - let mut next_round = Vec::new(); - for (offset, crc) in pending.iter().copied() { - match reader.resolve_at(offset) { - Ok((kind, payload)) => { - let oid = GitOid::compute_git_object(kind.name(), &payload); - if reader.by_oid.insert(oid, offset).is_some() { - return Err(corrupt(format!("pack holds object {oid} twice"))); - } - reader.entries.push(PackIndexEntry { - oid, - offset, - crc32: crc, - }); + // Whole objects and offset deltas resolve by position alone; an entry whose chain + // reaches a reference delta with an unresolved base waits on that base's ID and is + // retried once that ID resolves. + let mut waiting: HashMap> = HashMap::new(); + let mut ready: Vec<(u64, u32)> = offsets.into_iter().rev().collect(); + while let Some((offset, crc)) = ready.pop() { + match reader.try_resolve_at(offset)? { + Ok((kind, payload)) => { + let oid = GitOid::compute_git_object(kind.name(), &payload); + if reader.by_oid.insert(oid, offset).is_some() { + return Err(corrupt(format!("pack holds object {oid} twice"))); } - Err(e) if e.message.starts_with(MISSING_REF_BASE) => { - next_round.push((offset, crc)) + reader.entries.push(PackIndexEntry { + oid, + offset, + crc32: crc, + }); + if let Some(w) = waiting.remove(&oid) { + ready.extend(w); } - Err(e) => return Err(e), } + Err(base) => waiting.entry(base).or_default().push((offset, crc)), } - if next_round.len() == pending.len() { - return Err(corrupt( - "pack has reference deltas whose base is not in the pack (thin pack)", - )); - } - pending = next_round; + } + if !waiting.is_empty() { + return Err(corrupt( + "pack has reference deltas whose base is not in the pack (thin pack)", + )); } reader.entries.sort_by_key(|e| e.oid); Ok(reader) @@ -463,6 +486,17 @@ impl PackReader { } fn resolve_at(&mut self, offset: u64) -> SynapseResult<(ObjectKind, std::sync::Arc>)> { + self.try_resolve_at(offset)? + .map_err(|base| corrupt(format!("pack reference delta base not found: {base}"))) + } + + /// Resolve the entry at `offset`, or name the reference delta base its chain reaches that + /// is not (yet) indexed. + #[allow(clippy::type_complexity)] + fn try_resolve_at( + &mut self, + offset: u64, + ) -> SynapseResult>), GitOid>> { let body_end = self.data.len() - 20; let mut chain: Vec<(u64, Vec)> = Vec::new(); let mut at = offset; @@ -483,10 +517,9 @@ impl PackReader { at = base; } RawEntry::RefDelta(base_oid, delta) => { - let base = *self - .by_oid - .get(&base_oid) - .ok_or_else(|| corrupt(format!("{MISSING_REF_BASE} {base_oid}")))?; + let Some(&base) = self.by_oid.get(&base_oid) else { + return Ok(Err(base_oid)); + }; chain.push((at, delta)); at = base; } @@ -503,12 +536,10 @@ impl PackReader { )?); self.remember(at, kind, payload.clone()); } - Ok((kind, payload)) + Ok(Ok((kind, payload))) } } -const MISSING_REF_BASE: &str = "pack reference delta base not found:"; - fn read_size_encoding(data: &[u8], pos: &mut usize) -> SynapseResult { let mut value = 0u64; let mut shift = 0u32; @@ -735,6 +766,87 @@ pub fn apply_delta_limited( mod tests { use super::*; + fn refusal(r: SynapseResult, case: &str) -> SynapseError { + match r { + Ok(p) => panic!( + "{case}: indexed {} entries instead of refusing", + p.entries().len() + ), + Err(e) => e, + } + } + + /// A pack whose first entry is a reference delta on the blob stored after it (legal on + /// disk: the base is in the pack), or with the base left out (a thin pack). + fn ref_delta_before_base_pack(with_base: bool) -> (Vec, GitObject, GitObject) { + let base = GitObject::new(ObjectKind::Blob, b"hello base object\n".repeat(4)); + let mut result = base.payload.clone(); + result.push(b'!'); + let target = GitObject::new(ObjectKind::Blob, result.clone()); + // Sizes (one byte each, both under 128), copy the whole base, insert one byte. + let delta = vec![ + base.payload.len() as u8, + result.len() as u8, + 0x90, + base.payload.len() as u8, + 0x01, + b'!', + ]; + let mut pack = Vec::new(); + pack.extend_from_slice(PACK_SIGNATURE); + pack.extend_from_slice(&PACK_VERSION.to_be_bytes()); + pack.extend_from_slice(&(1 + u32::from(with_base)).to_be_bytes()); + encode_entry_header(OBJ_REF_DELTA, delta.len() as u64, &mut pack); + pack.extend_from_slice(&base.oid().0); + pack.extend_from_slice(&zlib_compress_store(&delta)); + if with_base { + encode_entry_header(base.kind.pack_type(), base.payload.len() as u64, &mut pack); + pack.extend_from_slice(&zlib_compress_store(&base.payload)); + } + let sum: [u8; 20] = Sha1::digest(&pack).into(); + pack.extend_from_slice(&sum); + (pack, base, target) + } + + #[test] + fn indexer_resolves_a_reference_delta_stored_before_its_base_within_the_budget() { + let (pack, base, target) = ref_delta_before_base_pack(true); + let declared = (base.payload.len() + target.payload.len()) as u64; + let fits = crate::repo::ReadLimits { + max_total_object_bytes: declared, + ..Default::default() + }; + let mut reader = PackReader::index_with_limits(pack.clone(), &fits).unwrap(); + assert_eq!(reader.entries().len(), 2); + assert_eq!(reader.read(base.oid()).unwrap().unwrap(), base); + assert_eq!(reader.read(target.oid()).unwrap().unwrap(), target); + // One byte short of what the entries declare: refused before anything is resolved. + let short = crate::repo::ReadLimits { + max_total_object_bytes: declared - 1, + ..Default::default() + }; + let err = refusal( + PackReader::index_with_limits(pack, &short), + "one byte short", + ); + assert_eq!(err.code, RefusalCode::QuotaPayloadTooLarge); + // The per-object cap applies to the delta result as well. + let (pack, _, _) = ref_delta_before_base_pack(true); + let capped = crate::repo::ReadLimits { + max_object_bytes: base.payload.len() as u64, + ..Default::default() + }; + let err = refusal( + PackReader::index_with_limits(pack, &capped), + "delta result over the cap", + ); + assert_eq!(err.code, RefusalCode::QuotaPayloadTooLarge); + let (thin, _, _) = ref_delta_before_base_pack(false); + let err = refusal(PackReader::index(thin), "thin pack"); + assert_eq!(err.code, RefusalCode::GitObjectCorrupt); + assert!(err.message.contains("thin pack"), "{}", err.message); + } + #[test] fn crc32_matches_the_ieee_check_value() { assert_eq!(crc32(b"123456789"), 0xcbf4_3926); diff --git a/crates/synapse-git/src/repo.rs b/crates/synapse-git/src/repo.rs index cbcfc6c..8a11e3a 100644 --- a/crates/synapse-git/src/repo.rs +++ b/crates/synapse-git/src/repo.rs @@ -316,7 +316,13 @@ impl GitRepository { } } let hex = oid.to_hex(); - let path = self.git_dir.join("objects").join(&hex[..2]).join(&hex[2..]); + // The fan-out directory must be a real directory too: `O_NOFOLLOW` covers only the + // last path component, so a linked `objects/ab` could lead the read outside. + let fanout = self.git_dir.join("objects").join(&hex[..2]); + if !real_dir(&fanout)? { + return Ok(None); + } + let path = fanout.join(&hex[2..]); // An incompressible object stored as zlib grows by 5 bytes per 65,535-byte block plus // 6; allow the object cap plus 1/64 and 1 KiB for that and the object header. let max_file = self.limits.max_object_bytes / 64 * 65 + 1024; diff --git a/crates/synapse-git/tests/common/mod.rs b/crates/synapse-git/tests/common/mod.rs new file mode 100644 index 0000000..2aedc8e --- /dev/null +++ b/crates/synapse-git/tests/common/mod.rs @@ -0,0 +1,292 @@ +//! Fixture helpers shared by the H1 tests (issue #48). The system `git` is used only as an +//! independent oracle and fixture builder. +#![allow(dead_code)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use synapse_core::GitOid; +use synapse_git::{GitHistory, GitObject}; + +pub fn temp_dir(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "synapse-h1-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Run `git -C dir args` with no user or system configuration and fixed identities and dates, +/// so fixture history is the same on every host. +pub fn git_raw(dir: &Path, args: &[&str], stdin: Option<&[u8]>) -> Output { + let mut cmd = Command::new("git"); + cmd.arg("-C") + .arg(dir) + .args(args) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_AUTHOR_NAME", "Fixture Author") + .env("GIT_AUTHOR_EMAIL", "author@fixture.invalid") + .env("GIT_AUTHOR_DATE", "1700000000 +0000") + .env("GIT_COMMITTER_NAME", "Fixture Committer") + .env("GIT_COMMITTER_EMAIL", "committer@fixture.invalid") + .env("GIT_COMMITTER_DATE", "1700000000 +0000"); + if let Some(input) = stdin { + use std::io::Write; + cmd.stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut child = cmd.spawn().unwrap(); + child.stdin.take().unwrap().write_all(input).unwrap(); + child.wait_with_output().unwrap() + } else { + cmd.output().unwrap() + } +} + +pub fn git(dir: &Path, args: &[&str]) -> Vec { + let out = git_raw(dir, args, None); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +pub fn git_text(dir: &Path, args: &[&str]) -> String { + String::from_utf8(git(dir, args)) + .unwrap() + .trim() + .to_string() +} + +pub fn write(dir: &Path, rel: &str, bytes: &[u8]) { + let path = dir.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); +} + +pub fn commit_all(dir: &Path, msg: &str) { + git(dir, &["add", "-A"]); + git(dir, &["commit", "-q", "--allow-empty", "-m", msg]); +} + +/// Deterministic pseudo-random bytes (xorshift), so the binary blob is incompressible. +pub fn noise(len: usize, seed: u64) -> Vec { + let mut x = seed | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} + +/// Real Git history with every shape the bridge must carry: nested trees, executable files, +/// a symlink, an empty file, an incompressible binary, a large compressible file revised many +/// times (delta material), a two-parent merge, an octopus merge, an empty commit, annotated +/// tags of a commit, a tree, a blob and another tag, a lightweight tag, and a commit with a +/// multi-line signature header and an `encoding` header. +pub fn build_fixture(dir: &Path) { + git(dir, &["init", "-q", "-b", "main"]); + let mut big = String::new(); + for i in 0..6000 { + big.push_str(&format!( + "line {i}: the quick brown fox jumps over the lazy dog\n" + )); + } + write(dir, "README.md", b"# fixture\n"); + write(dir, "src/lib.rs", b"pub fn one() -> u32 { 1 }\n"); + write(dir, "src/deep/nested/mod.rs", b"// nested\n"); + write(dir, "data/empty.txt", b""); + write(dir, "data/noise.bin", &noise(40_000, 7)); + write(dir, "docs/big.txt", big.as_bytes()); + write(dir, "tools/run.sh", b"#!/bin/sh\necho run\n"); + let mut perms = fs::metadata(dir.join("tools/run.sh")) + .unwrap() + .permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755); + fs::set_permissions(dir.join("tools/run.sh"), perms).unwrap(); + std::os::unix::fs::symlink("../README.md", dir.join("docs/readme-link")).unwrap(); + commit_all(dir, "initial"); + for rev in 0..12 { + let edited = big.replace( + &format!("line {}:", rev * 400), + &format!("LINE {} (rev {rev}):", rev * 400), + ); + write(dir, "docs/big.txt", edited.as_bytes()); + write( + dir, + "src/lib.rs", + format!("pub fn one() -> u32 {{ {rev} }}\n").as_bytes(), + ); + commit_all(dir, &format!("revision {rev}")); + } + git(dir, &["tag", "-a", "v1", "-m", "release v1"]); + git(dir, &["tag", "light"]); + git(dir, &["checkout", "-q", "-b", "feature"]); + write(dir, "src/feature.rs", b"pub fn feature() {}\n"); + commit_all(dir, "feature work"); + git(dir, &["checkout", "-q", "main"]); + write(dir, "src/main_only.rs", b"fn main_only() {}\n"); + commit_all(dir, "main work"); + git( + dir, + &["merge", "-q", "--no-ff", "feature", "-m", "merge feature"], + ); + for name in ["oct-a", "oct-b"] { + git(dir, &["checkout", "-q", "-b", name, "main"]); + write(dir, &format!("{name}.txt"), name.as_bytes()); + commit_all(dir, name); + } + git(dir, &["checkout", "-q", "main"]); + git( + dir, + &["merge", "-q", "--no-ff", "oct-a", "oct-b", "-m", "octopus"], + ); + commit_all(dir, "empty commit"); + let tree = git_text(dir, &["rev-parse", "HEAD^{tree}"]); + let blob = git_text(dir, &["rev-parse", "HEAD:README.md"]); + git( + dir, + &["tag", "-a", "tree-tag", &tree, "-m", "a tag of a tree"], + ); + git( + dir, + &["tag", "-a", "blob-tag", &blob, "-m", "a tag of a blob"], + ); + git( + dir, + &[ + "tag", + "-a", + "-f", + "nested-tag", + "v1", + "-m", + "a tag of a tag", + ], + ); + // Objects reachable only through a tag: a blob in no tree, and an inner tag whose own + // reference is deleted, so only the outer tag reaches it. + let out = git_raw( + dir, + &["hash-object", "-w", "--stdin"], + Some(b"reachable only through a tag\n"), + ); + let lone_blob = String::from_utf8(out.stdout).unwrap().trim().to_string(); + git( + dir, + &[ + "tag", + "-a", + "lone-blob-tag", + &lone_blob, + "-m", + "tag of a lone blob", + ], + ); + git(dir, &["tag", "-a", "inner", "v1", "-m", "inner tag"]); + git( + dir, + &["tag", "-a", "-f", "outer-only", "inner", "-m", "outer tag"], + ); + git(dir, &["tag", "-d", "inner"]); + // A commit with a multi-line signature header and an encoding header, written as raw + // bytes (no signing key is involved: the header is opaque data to Git and to the bridge). + let head = git_text(dir, &["rev-parse", "HEAD"]); + let raw = format!( + "tree {tree}\nparent {head}\n\ +author Fixture Author 1700000001 +0100\n\ +committer Fixture Committer 1700000002 -0230\n\ +encoding ISO-8859-1\n\ +gpgsig -----BEGIN PGP SIGNATURE-----\n \n iQEzBAABCAAdFiEEfixture\n =abcd\n -----END PGP SIGNATURE-----\n\ +\nsigned commit with caf\u{e9}\n" + ); + let out = git_raw( + dir, + &["hash-object", "-t", "commit", "-w", "--stdin"], + Some(raw.as_bytes()), + ); + assert!(out.status.success()); + let signed = String::from_utf8(out.stdout).unwrap().trim().to_string(); + git(dir, &["update-ref", "refs/heads/signed", &signed]); +} + +/// `git cat-file --batch-all-objects --batch`: every object in the database as (kind, payload). +pub fn git_all_objects(dir: &Path) -> BTreeMap)> { + let out = git(dir, &["cat-file", "--batch-all-objects", "--batch"]); + let mut objects = BTreeMap::new(); + let mut rest = &out[..]; + while !rest.is_empty() { + let nl = rest.iter().position(|&b| b == b'\n').unwrap(); + let header = std::str::from_utf8(&rest[..nl]).unwrap(); + let mut parts = header.split(' '); + let oid = GitOid::from_hex(parts.next().unwrap()).unwrap(); + let kind = parts.next().unwrap().to_string(); + let size: usize = parts.next().unwrap().parse().unwrap(); + let payload = rest[nl + 1..nl + 1 + size].to_vec(); + objects.insert(oid, (kind, payload)); + rest = &rest[nl + 1 + size + 1..]; + } + objects +} + +pub fn git_refs(dir: &Path) -> BTreeMap { + git_text(dir, &["for-each-ref", "--format=%(refname) %(objectname)"]) + .lines() + .map(|l| { + let (name, oid) = l.split_once(' ').unwrap(); + (name.to_string(), GitOid::from_hex(oid).unwrap()) + }) + .collect() +} + +pub fn assert_history_matches_git(history: &GitHistory, dir: &Path) { + let oracle = git_all_objects(dir); + let native: BTreeSet = history.objects.keys().copied().collect(); + let expected: BTreeSet = oracle.keys().copied().collect(); + assert_eq!( + native, expected, + "native reachable set != git object database" + ); + for (oid, object) in &history.objects { + let (kind, payload) = &oracle[oid]; + assert_eq!(object.kind.name(), kind, "kind of {oid}"); + assert_eq!(&object.payload, payload, "payload of {oid}"); + assert_eq!(object.oid(), *oid); + } + assert_eq!(history.refs, git_refs(dir), "references"); +} + +pub fn assert_fsck_strict_clean(dir: &Path) { + let out = git_raw(dir, &["fsck", "--full", "--strict", "--no-dangling"], None); + let stderr = String::from_utf8_lossy(&out.stderr); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + out.status.success() && !stderr.contains("error") && !stderr.contains("warning"), + "git fsck --full --strict failed in {}:\n{stdout}\n{stderr}", + dir.display() + ); +} + +/// Number of entries whose pack type (`gitformat-pack`: 3 bits after the MSB of the first byte) is `t`. +pub fn count_entry_type(pack: &[u8], entries: &[synapse_git::PackIndexEntry], t: u8) -> usize { + entries + .iter() + .filter(|e| (pack[e.offset as usize] >> 4) & 0x07 == t) + .count() +} + +pub fn objects_of(history: &GitHistory) -> Vec { + history.objects.values().cloned().collect() +} diff --git a/crates/synapse-git/tests/h1_bridge_import.rs b/crates/synapse-git/tests/h1_bridge_import.rs new file mode 100644 index 0000000..7cba028 --- /dev/null +++ b/crates/synapse-git/tests/h1_bridge_import.rs @@ -0,0 +1,982 @@ +//! ADR 0003 step H1 (issue #48), cut 2: the persistent `GitOid` map, native import into the +//! store (WAL kind 10), export from the stored state with identical Git object IDs, and +//! deterministic export of native revisions. The system `git` builds fixture history and +//! checks the exported repositories (`cat-file`, `for-each-ref`, `fsck --full --strict`); it is +//! never used by the bridge itself. + +mod common; + +use common::*; +use ed25519_dalek::SigningKey; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use synapse_core::{ + BlobDigest, ChainPath, ChangeId, ChangesetEnvelopeV2, ChangesetId, FileMode, GitOid, + IdempotencyKey, MutationOp, PathMutation, RefusalCode, RepoId, RevisionContent, SynapseResult, + TreeDigest, WalSeq, +}; +use synapse_git::{export_store, import_repository, ImportReceipt, ReadLimits}; +use synapse_store::git_bridge::{ + GitBlobMapping, GitBridgeEntry, GitBridgeTarget, GitImportedCommit, GitImportedRefs, + GitTreeMapping, +}; +use synapse_store::{ + AdmissionAuthority, FsyncPolicy, RecoveryPolicy, RevisionOutcome, StoreLimits, SynapseStore, +}; + +fn limits() -> StoreLimits { + StoreLimits { + wal_prealloc_bytes: 64 * 1024, + ..Default::default() + } +} + +fn open_store(dir: &Path, limits: StoreLimits) -> SynapseStore { + SynapseStore::open( + dir, + FsyncPolicy::EveryCommit, + RecoveryPolicy::StrictFailClosed, + limits, + ) + .unwrap() +} + +fn repo_id() -> RepoId { + RepoId::new("aien-dev/h1-fixture").unwrap() +} + +/// A 32-byte value derived from a label (keys and idempotency keys in these tests are never +/// literal constants). +fn derived(label: &str) -> [u8; 32] { + Sha256::digest(label.as_bytes()).into() +} + +fn import(store: &mut SynapseStore, git_dir: &Path) -> SynapseResult { + import_repository(store, &repo_id(), git_dir, ReadLimits::default()) +} + +fn export(store: &SynapseStore, native: &BTreeMap, dest: &Path) -> GitOid { + export_store(store, &repo_id(), native, dest) + .unwrap_or_else(|e| panic!("export refused: {e}")) + .pack_checksum +} + +fn object_ids(dir: &Path) -> BTreeSet { + git_all_objects(dir).keys().copied().collect() +} + +/// The fixture repository with every object in one pack (as a clone would hold it). +fn fixture(label: &str) -> PathBuf { + let dir = temp_dir(label); + build_fixture(&dir); + git(&dir, &["repack", "-a", "-d", "-q"]); + dir +} + +#[test] +fn h1_import_then_export_from_the_store_gives_identical_ids_refs_and_fsck_strict_clean() { + let src = fixture("c2-round"); + let root = temp_dir("c2-round-out"); + let mut store = open_store(&root.join("store"), limits()); + let receipt = import(&mut store, &src.join(".git")).expect("import"); + assert!(receipt.commits >= 20 && receipt.tags >= 6, "{receipt:?}"); + assert_eq!(receipt.records_skipped, 0); + let dest = root.join("export.git"); + export(&store, &BTreeMap::new(), &dest); + assert_fsck_strict_clean(&dest); + let source_ids = object_ids(&src); + assert_eq!( + object_ids(&dest), + source_ids, + "every object, identical Git IDs" + ); + for (oid, (kind, payload)) in git_all_objects(&dest) { + let (k, p) = &git_all_objects(&src)[&oid]; + assert_eq!((&kind, &payload), (k, p), "object {oid}"); + } + assert_eq!(git_refs(&dest), git_refs(&src), "references"); + assert_eq!( + git_text(&dest, &["symbolic-ref", "HEAD"]), + git_text(&src, &["symbolic-ref", "HEAD"]) + ); + // The persistent map: every commit to (ChangesetId, TreeDigest), the tree digest being + // the one its Git tree maps to, and every parent an imported revision. + let bridge = store.git_bridge_repo(&repo_id()).expect("bridge map"); + let mut commits = 0; + for (oid, (kind, payload)) in git_all_objects(&src) { + if kind != "commit" { + continue; + } + commits += 1; + let (id, tree) = bridge.commit(oid).expect("commit mapped"); + let text = String::from_utf8_lossy(&payload).to_string(); + let tree_oid = GitOid::from_hex(&text[5..45]).unwrap(); + assert_eq!(bridge.tree(tree_oid), Some(tree), "tree of {oid}"); + let c = bridge.commit_by_changeset(id).expect("reverse map"); + assert_eq!(c.git_oid, oid); + } + assert_eq!(commits, receipt.commits); + // The same history imported again writes nothing. + let again = import(&mut store, &src.join(".git")).expect("re-import"); + assert_eq!(again.records_written, 0, "{again:?}"); + fs::remove_dir_all(&root).unwrap(); + fs::remove_dir_all(&src).unwrap(); +} + +#[test] +fn h1_bridge_map_survives_reopen_and_checkpoint_with_the_same_export() { + let src = fixture("c2-reopen"); + let root = temp_dir("c2-reopen-out"); + let store_dir = root.join("store"); + let mut store = open_store(&store_dir, limits()); + import(&mut store, &src.join(".git")).expect("import"); + let first = export(&store, &BTreeMap::new(), &root.join("a.git")); + let state_root = store.state_root_digest(None).unwrap(); + drop(store); + let mut store = open_store(&store_dir, limits()); + assert_eq!(store.state_root_digest(None).unwrap(), state_root); + assert_eq!( + export(&store, &BTreeMap::new(), &root.join("b.git")), + first, + "replay rebuilds the same map" + ); + store.checkpoint_and_compact(10_000).expect("checkpoint"); + assert_eq!( + export(&store, &BTreeMap::new(), &root.join("c.git")), + first, + "compaction keeps the bridge blobs and records" + ); + drop(store); + let store = open_store(&store_dir, limits()); + assert_eq!(store.state_root_digest(None).unwrap(), state_root); + assert_eq!( + export(&store, &BTreeMap::new(), &root.join("d.git")), + first, + "a checkpoint load rebuilds the same map" + ); + assert_fsck_strict_clean(&root.join("d.git")); + fs::remove_dir_all(&root).unwrap(); + fs::remove_dir_all(&src).unwrap(); +} + +/// Admit one native revision on top of the imported `main` tip: it edits a file, deletes a +/// file, and turns `README.md` into a directory. +fn upsert(path: &str, body: &[u8]) -> PathMutation { + PathMutation { + path: path.into(), + op: MutationOp::Upsert { + mode: FileMode::Regular644, + blob_digest: BlobDigest::compute(body), + size_bytes: body.len() as u64, + }, + } +} + +/// Admit one native revision (keys, change and idempotency key derived from `label`). +fn admit( + store: &mut SynapseStore, + parent: ChangesetId, + base_tree: TreeDigest, + mutations: Vec, + bodies: &[&[u8]], + label: &str, +) -> ChangesetId { + let content = RevisionContent { + change_id: ChangeId( + derived(&format!("{label} change"))[..16] + .try_into() + .unwrap(), + ), + base_tree, + parents: vec![parent], + mutations, + intent_summary: format!("{label} on imported history"), + }; + let author = SigningKey::from_bytes(&derived(&format!("{label} author"))); + let env = ChangesetEnvelopeV2::sign_new( + &author, + repo_id(), + content, + IdempotencyKey(derived(&format!("{label} idempotency"))), + 1_000, + ); + let blobs: Vec<(BlobDigest, Vec)> = bodies + .iter() + .map(|b| (BlobDigest::compute(b), b.to_vec())) + .collect(); + let chain = ChainPath::new(vec![derived("h1 chain root"), derived("h1 chain agent")]).unwrap(); + let authority = AdmissionAuthority { + chain, + change_scope: Vec::new(), + }; + match store.admit_revision(env, &blobs, authority, 5_000).unwrap() { + RevisionOutcome::Admitted { changeset_id, .. } => changeset_id, + other => panic!("expected an admission, got {other:?}"), + } +} + +/// Admit one native revision on top of the imported `main` tip: it edits a file, deletes a +/// file, and turns `README.md` into a directory. +fn admit_native(store: &mut SynapseStore, src: &Path) -> ChangesetId { + let tip = GitOid::from_hex(&git_text(src, &["rev-parse", "refs/heads/main"])).unwrap(); + let (parent, base_tree) = store + .git_bridge_repo(&repo_id()) + .unwrap() + .commit(tip) + .unwrap(); + let lib: &[u8] = b"pub fn one() -> u32 { 42 }\n"; + let inner: &[u8] = b"now a directory\n"; + let mutations = vec![ + upsert("README.md/inner.txt", inner), + PathMutation { + path: "data/empty.txt".into(), + op: MutationOp::Delete, + }, + upsert("src/lib.rs", lib), + ]; + admit( + store, + parent, + base_tree, + mutations, + &[inner, lib], + "h1 native", + ) +} + +#[test] +fn h1_a_native_child_of_a_native_revision_exports_on_its_parents_result_tree() { + let src = fixture("c2-chain"); + let root = temp_dir("c2-chain-out"); + let mut store = open_store(&root.join("store"), limits()); + import(&mut store, &src.join(".git")).expect("import"); + let first = admit_native(&mut store, &src); + let one = BTreeMap::from([("refs/heads/agent".to_string(), first)]); + let a = root.join("a.git"); + export(&store, &one, &a); + // The first revision's result tree, as an independent path computes it: the import of + // the exported repository converts its Git tree to a TreeDigest. + let mut other = open_store(&root.join("other"), limits()); + import(&mut other, &a).expect("re-import of the export"); + let tip = GitOid::from_hex(&git_text(&a, &["rev-parse", "refs/heads/agent"])).unwrap(); + let (_, result) = other + .git_bridge_repo(&repo_id()) + .unwrap() + .commit(tip) + .unwrap(); + drop(other); + let body: &[u8] = b"second\n"; + let second = admit( + &mut store, + first, + result, + vec![upsert("docs/second.md", body)], + &[body], + "h1 native child", + ); + let both = BTreeMap::from([ + ("refs/heads/agent".to_string(), first), + ("refs/heads/agent2".to_string(), second), + ]); + let b = root.join("b.git"); + export(&store, &both, &b); + assert_fsck_strict_clean(&b); + assert_eq!( + git_text(&b, &["rev-parse", "refs/heads/agent2^"]), + tip.to_hex(), + "the child's parent is the first native commit, unchanged" + ); + let diff = git_text( + &b, + &[ + "diff-tree", + "-r", + "--name-status", + "refs/heads/agent", + "refs/heads/agent2", + ], + ); + assert_eq!( + diff, "A\tdocs/second.md", + "the child adds exactly its one file" + ); + fs::remove_dir_all(&root).unwrap(); + fs::remove_dir_all(&src).unwrap(); +} + +#[test] +fn h1_native_revisions_export_deterministically_on_imported_history() { + let src = fixture("c2-native"); + let root = temp_dir("c2-native-out"); + let store_dir = root.join("store"); + let mut store = open_store(&store_dir, limits()); + import(&mut store, &src.join(".git")).expect("import"); + let id = admit_native(&mut store, &src); + let native = BTreeMap::from([("refs/heads/agent".to_string(), id)]); + let a = export(&store, &native, &root.join("a.git")); + let b = export(&store, &native, &root.join("b.git")); + assert_eq!(a, b, "two exports give the same pack"); + drop(store); + let store = open_store(&store_dir, limits()); + assert_eq!( + export(&store, &native, &root.join("c.git")), + a, + "the same pack after a reopen" + ); + let dest = root.join("a.git"); + assert_fsck_strict_clean(&dest); + let main = git_text(&src, &["rev-parse", "refs/heads/main"]); + let commit = git_text(&dest, &["cat-file", "-p", "refs/heads/agent"]); + assert!(commit.contains(&format!("\nparent {main}\n")), "{commit}"); + assert!( + commit.contains(" 0 +0000\n"), + "{commit}" + ); + assert!( + commit.contains("committer Synapse Engine 0 +0000\n"), + "{commit}" + ); + assert!( + commit.contains(&format!("Synapse-Changeset-Id: {}", id.to_hex())), + "{commit}" + ); + let files = git_text(&dest, &["ls-tree", "-r", "--name-only", "refs/heads/agent"]); + let files: BTreeSet<&str> = files.lines().collect(); + assert!(files.contains("README.md/inner.txt")); + assert!(!files.contains("README.md") && !files.contains("data/empty.txt")); + assert_eq!( + git_text(&dest, &["show", "refs/heads/agent:src/lib.rs"]), + "pub fn one() -> u32 { 42 }" + ); + // Everything else is the parent's tree, unchanged. + let parent_files = git_text(&src, &["ls-tree", "-r", "--name-only", "refs/heads/main"]); + for f in parent_files.lines() { + if f != "README.md" && f != "data/empty.txt" { + assert!(files.contains(f), "{f} kept"); + } + } + // Imported references are unchanged beside the native one. + let mut expected = git_refs(&src); + expected.insert( + "refs/heads/agent".into(), + GitOid::from_hex(&git_text(&dest, &["rev-parse", "refs/heads/agent"])).unwrap(), + ); + assert_eq!(git_refs(&dest), expected); + fs::remove_dir_all(&root).unwrap(); + fs::remove_dir_all(&src).unwrap(); +} + +// --------------------------------------------------------------------------------------- +// Hostile input at the import boundary: refused typed, before anything is written. +// --------------------------------------------------------------------------------------- + +/// A small repository whose `main` holds one commit of `tree` (raw tree bytes written with +/// `hash-object --literally`, which accepts what Git itself would never write). +fn repo_with_raw_tree(label: &str, raw_tree: &[u8]) -> PathBuf { + let dir = temp_dir(label); + git(&dir, &["init", "-q", "-b", "main"]); + let out = git_raw( + &dir, + &["hash-object", "-t", "tree", "--literally", "-w", "--stdin"], + Some(raw_tree), + ); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let tree = String::from_utf8(out.stdout).unwrap().trim().to_string(); + let commit = git_text(&dir, &["commit-tree", &tree, "-m", "crafted"]); + git(&dir, &["update-ref", "refs/heads/main", &commit]); + dir +} + +fn blob(dir: &Path, body: &[u8]) -> [u8; 20] { + let out = git_raw(dir, &["hash-object", "-w", "--stdin"], Some(body)); + GitOid::from_hex(String::from_utf8(out.stdout).unwrap().trim()) + .unwrap() + .0 +} + +fn tree_entry(mode: &str, name: &[u8], oid: [u8; 20]) -> Vec { + let mut e = format!("{mode} ").into_bytes(); + e.extend_from_slice(name); + e.push(0); + e.extend_from_slice(&oid); + e +} + +/// Import `git_dir` into a fresh store and require refusal with `code` and an untouched store. +fn assert_import_refused(git_dir: &Path, limits: StoreLimits, code: RefusalCode, case: &str) { + let root = temp_dir("c2-refused-store"); + let mut store = open_store(&root, limits); + let before = store.committed_seq(); + match import(&mut store, git_dir) { + Err(e) => assert_eq!(e.code, code, "{case}: wrong refusal: {e}"), + Ok(r) => panic!("{case}: imported instead of refusing: {r:?}"), + } + assert_eq!(store.committed_seq(), before, "{case}: nothing written"); + assert!(store.git_bridge_repo(&repo_id()).is_none(), "{case}"); + drop(store); + fs::remove_dir_all(&root).unwrap(); +} + +#[test] +fn h1_import_refuses_trees_with_no_native_form_before_writing() { + let scratch = temp_dir("c2-blobs"); + git(&scratch, &["init", "-q"]); + let a = blob(&scratch, b"a\n"); + let cases: Vec<(&str, Vec)> = vec![ + ("legacy mode 100664", tree_entry("100664", b"a.txt", a)), + ("gitlink 160000", tree_entry("160000", b"sub", a)), + ("name .git", tree_entry("100644", b".git", a)), + ("name not UTF-8", tree_entry("100644", b"caf\xe9", a)), + ( + "entries out of order", + [tree_entry("100644", b"b", a), tree_entry("100644", b"a", a)].concat(), + ), + ( + "duplicate names", + [tree_entry("100644", b"a", a), tree_entry("100644", b"a", a)].concat(), + ), + ]; + for (case, raw) in cases { + let dir = repo_with_raw_tree("c2-raw-tree", &raw); + // The gitlink names no object in this repository (the reader skips gitlinks); every + // other entry names the blob, written into this repository too. + blob(&dir, b"a\n"); + assert_import_refused( + &dir.join(".git"), + limits(), + RefusalCode::GitObjectNotRepresentable, + case, + ); + fs::remove_dir_all(&dir).unwrap(); + } + // A zero-padded directory mode (`040000`) is a subtree Git would write as `40000`. + let dir = temp_dir("c2-mode-040000"); + git(&dir, &["init", "-q", "-b", "main"]); + let inner = repo_raw_tree_object(&dir, &tree_entry("100644", b"a", blob(&dir, b"a\n"))); + let outer = repo_raw_tree_object(&dir, &tree_entry("040000", b"d", inner)); + let commit = git_text(&dir, &["commit-tree", &GitOid(outer).to_hex(), "-m", "x"]); + git(&dir, &["update-ref", "refs/heads/main", &commit]); + assert_import_refused( + &dir.join(".git"), + limits(), + RefusalCode::GitObjectNotRepresentable, + "directory mode 040000", + ); + fs::remove_dir_all(&dir).unwrap(); + fs::remove_dir_all(&scratch).unwrap(); +} + +fn repo_raw_tree_object(dir: &Path, raw: &[u8]) -> [u8; 20] { + let out = git_raw( + dir, + &["hash-object", "-t", "tree", "--literally", "-w", "--stdin"], + Some(raw), + ); + assert!(out.status.success()); + GitOid::from_hex(String::from_utf8(out.stdout).unwrap().trim()) + .unwrap() + .0 +} + +#[test] +fn h1_import_refuses_reference_names_that_would_write_outside_before_writing() { + let src = temp_dir("c2-ref-escape"); + git(&src, &["init", "-q", "-b", "main"]); + write(&src, "a.txt", b"a\n"); + commit_all(&src, "one"); + let tip = git_text(&src, &["rev-parse", "HEAD"]); + // The repository reader refuses a malformed reference name (`GitObjectCorrupt`, cut 1); + // the bridge refuses names that are valid alone but collide on disk (`PathInvalid`). + let corrupt = RefusalCode::GitObjectCorrupt; + for (case, lines, code) in [ + ( + "dot-dot path", + format!("{tip} refs/heads/../../../escape\n"), + corrupt, + ), + ( + "absolute-looking", + format!("{tip} refs//heads/x\n"), + corrupt, + ), + ( + "file and directory", + format!("{tip} refs/heads/a\n{tip} refs/heads/a/b\n"), + RefusalCode::PathInvalid, + ), + ("outside refs/", format!("{tip} HEAD2\n"), corrupt), + ] { + fs::write(src.join(".git/packed-refs"), lines).unwrap(); + assert_import_refused(&src.join(".git"), limits(), code, case); + } + assert!(!src.join("escape").exists() && !src.parent().unwrap().join("escape").exists()); + fs::remove_dir_all(&src).unwrap(); +} + +#[test] +fn h1_import_refuses_a_blob_over_the_store_cap_before_writing() { + let src = temp_dir("c2-big-blob"); + git(&src, &["init", "-q", "-b", "main"]); + write(&src, "small.txt", b"small\n"); + write(&src, "big.bin", &noise(4096, 3)); + commit_all(&src, "one"); + let small = StoreLimits { + max_blob_bytes: 4095, + ..limits() + }; + assert_import_refused( + &src.join(".git"), + small, + RefusalCode::QuotaPayloadTooLarge, + "blob over max_blob_bytes", + ); + fs::remove_dir_all(&src).unwrap(); +} + +/// Crafted kind 10 entries handed straight to the store (as a replica or a buggy importer +/// could): each is refused typed and nothing is written. +#[test] +fn h1_store_refuses_crafted_bridge_entries() { + let root = temp_dir("c2-crafted"); + let mut store = open_store(&root, limits()); + let repo = repo_id(); + let body = b"hello\n".as_slice(); + let digest = BlobDigest::compute(body); + let oid = GitOid::compute_blob(body); + let mapping = GitBlobMapping { + git_oid: oid, + blob: digest, + size_bytes: body.len() as u64, + }; + let objects = |blobs: Vec, trees: Vec| { + GitBridgeEntry::Objects { blobs, trees } + }; + let seq: WalSeq = store + .record_git_import( + &repo, + objects(vec![mapping.clone()], vec![]), + &[(digest, body)], + ) + .unwrap() + .expect("written"); + let refused = |store: &mut SynapseStore, + entry: GitBridgeEntry, + blobs: &[(BlobDigest, &[u8])], + code: RefusalCode, + case: &str| { + match store.record_git_import(&repo, entry, blobs) { + Err(e) => assert_eq!(e.code, code, "{case}: {e}"), + Ok(s) => panic!("{case}: accepted ({s:?})"), + } + assert_eq!(store.committed_seq(), seq, "{case}: nothing written"); + }; + // The same Git blob remapped to other bytes. + let other = b"other\n".as_slice(); + let remap = GitBlobMapping { + blob: BlobDigest::compute(other), + size_bytes: other.len() as u64, + ..mapping.clone() + }; + refused( + &mut store, + objects(vec![remap], vec![]), + &[(BlobDigest::compute(other), other)], + RefusalCode::GitObjectCorrupt, + "Git ID remapped", + ); + // A size that is not the blob's. + let wrong_size = GitBlobMapping { + git_oid: GitOid::compute_blob(other), + blob: BlobDigest::compute(other), + size_bytes: 99, + }; + refused( + &mut store, + objects(vec![wrong_size], vec![]), + &[(BlobDigest::compute(other), other)], + RefusalCode::BlobDigestMismatch, + "declared size", + ); + // A blob neither provided nor stored. + let missing = GitBlobMapping { + git_oid: GitOid::compute_blob(other), + blob: BlobDigest::compute(other), + size_bytes: other.len() as u64, + }; + refused( + &mut store, + objects(vec![missing], vec![]), + &[], + RefusalCode::BlobNotFound, + "missing blob", + ); + // Bytes that do not hash to their declared digest. + refused( + &mut store, + objects(vec![mapping.clone()], vec![]), + &[(digest, other)], + RefusalCode::BlobDigestMismatch, + "blob bytes", + ); + // A tree node that is not canonical (one declared entry, none present). + let bad_node = GitTreeMapping { + git_oid: GitOid::compute_git_object("tree", b"x"), + node: 1u32.to_le_bytes().to_vec(), + }; + refused( + &mut store, + objects(vec![], vec![bad_node]), + &[], + RefusalCode::TreeNodeMalformed, + "malformed node", + ); + // A commit on a tree or parent that was never imported. + let commit = |tree: TreeDigest, parents: Vec| { + GitBridgeEntry::Commit(GitImportedCommit { + git_oid: GitOid::compute_git_object("commit", b"crafted"), + tree, + parents, + headers: b"author a 0 +0000\ncommitter a 0 +0000\n\nm\n".to_vec(), + }) + }; + refused( + &mut store, + commit(TreeDigest(derived("unknown tree")), vec![]), + &[], + RefusalCode::TreeNodeNotFound, + "unknown tree", + ); + refused( + &mut store, + commit( + TreeDigest::empty(), + vec![ChangesetId(derived("unknown parent"))], + ), + &[], + RefusalCode::ParentChangesetNotFound, + "unknown parent", + ); + // References: a name that escapes, and a target never imported. + let refs = |name: &str, target: GitBridgeTarget| { + GitBridgeEntry::Refs(GitImportedRefs { + refs: vec![(name.to_string(), target)], + head: None, + }) + }; + refused( + &mut store, + refs("refs/heads/../../x", GitBridgeTarget::Blob(digest)), + &[], + RefusalCode::PathInvalid, + "ref name", + ); + refused( + &mut store, + refs( + "refs/heads/x", + GitBridgeTarget::Commit(ChangesetId(derived("never imported"))), + ), + &[], + RefusalCode::GitObjectCorrupt, + "ref target", + ); + drop(store); + fs::remove_dir_all(&root).unwrap(); +} + +/// Review of #96, B1: every record is checked against the store's limits (WAL frame cap, +/// changeset bytes) before the first write, so a record the store would refuse late leaves +/// the store untouched instead of half imported. +#[test] +fn h1_import_refuses_a_record_over_the_store_limits_before_writing() { + // One commit whose message alone is larger than the WAL frame cap. + let src = temp_dir("c2-huge-commit"); + git(&src, &["init", "-q", "-b", "main"]); + write(&src, "a.txt", b"a\n"); + git(&src, &["add", "-A"]); + let tree = git_text(&src, &["write-tree"]); + let message = "m".repeat(100 * 1024); + let out = git_raw( + &src, + &["commit-tree", &tree, "-F", "-"], + Some(message.as_bytes()), + ); + assert!(out.status.success()); + let commit = String::from_utf8(out.stdout).unwrap().trim().to_string(); + git(&src, &["update-ref", "refs/heads/main", &commit]); + let tight = StoreLimits { + max_blob_bytes: 4 * 1024, + max_total_changeset_bytes: 16 * 1024, + max_wal_record_bytes: 64 * 1024, + ..limits() + }; + assert_import_refused( + &src.join(".git"), + tight.clone(), + RefusalCode::QuotaPayloadTooLarge, + "commit record over max_wal_record_bytes", + ); + // The same store, refused once under the tight limits, then imports the repository + // in full under the default limits and exports it with identical IDs. + let root = temp_dir("c2-huge-commit-store"); + { + let mut store = open_store(&root, tight); + let refusal = import(&mut store, &src.join(".git")).expect_err("over the frame cap"); + assert_eq!(refusal.code, RefusalCode::QuotaPayloadTooLarge, "{refusal}"); + assert_eq!(store.committed_seq(), WalSeq(0)); + } + let mut store = open_store(&root, limits()); + import(&mut store, &src.join(".git")).expect("import under the default limits"); + let dest = root.join("export.git"); + export(&store, &BTreeMap::new(), &dest); + assert_fsck_strict_clean(&dest); + assert_eq!(object_ids(&dest), object_ids(&src)); + drop(store); + fs::remove_dir_all(&root).unwrap(); + fs::remove_dir_all(&src).unwrap(); + // A blob within max_blob_bytes but over max_total_changeset_bytes (a custom config). + let src = temp_dir("c2-blob-over-changeset"); + git(&src, &["init", "-q", "-b", "main"]); + write(&src, "small.txt", b"small\n"); + write(&src, "mid.bin", &noise(6 * 1024, 5)); + commit_all(&src, "one"); + let skewed = StoreLimits { + max_blob_bytes: 8 * 1024, + max_total_changeset_bytes: 4 * 1024, + ..limits() + }; + assert_import_refused( + &src.join(".git"), + skewed, + RefusalCode::QuotaPayloadTooLarge, + "blob over max_total_changeset_bytes", + ); + fs::remove_dir_all(&src).unwrap(); +} + +/// Review of #96, B2: the store checks every Git ID it is handed against the content it +/// maps (re-hashed with the workspace SHA-1), and refuses a Git ID twice in one entry, so a +/// crafted record cannot poison the map; a correct import still works afterwards. +#[test] +fn h1_store_refuses_git_ids_that_do_not_match_their_content() { + let root = temp_dir("c2-poison"); + let mut store = open_store(&root.join("store"), limits()); + let repo = RepoId::new("aien-dev/h1-poison").unwrap(); + let a = b"alpha\n".as_slice(); + let b = b"bravo\n".as_slice(); + let (da, db) = (BlobDigest::compute(a), BlobDigest::compute(b)); + let x = GitOid::compute_blob(a); + let map = |oid: GitOid, blob: BlobDigest, len: usize| GitBlobMapping { + git_oid: oid, + blob, + size_bytes: len as u64, + }; + let refused = |store: &mut SynapseStore, + entry: GitBridgeEntry, + blobs: &[(BlobDigest, &[u8])], + case: &str| { + match store.record_git_import(&repo, entry, blobs) { + Err(e) => assert_eq!(e.code, RefusalCode::GitObjectCorrupt, "{case}: {e}"), + Ok(s) => panic!("{case}: accepted ({s:?})"), + } + assert_eq!(store.committed_seq(), WalSeq(0), "{case}: nothing written"); + assert!(store.git_bridge_repo(&repo).is_none(), "{case}: no map"); + }; + refused( + &mut store, + GitBridgeEntry::Objects { + blobs: vec![map(x, da, a.len()), map(x, db, b.len())], + trees: vec![], + }, + &[(da, a), (db, b)], + "one Git ID mapped to two blobs in one entry", + ); + refused( + &mut store, + GitBridgeEntry::Objects { + blobs: vec![map(x, db, b.len())], + trees: vec![], + }, + &[(db, b)], + "a blob under another blob's Git ID", + ); + // A tree under a wrong Git ID (its node names blob `a`, mapped in the same entry). + let mut enc = synapse_core::TreeNodeEncoder::new(1); + enc.file("a.txt", FileMode::Regular644, &da, a.len() as u64) + .unwrap(); + let (tree_digest, node) = enc.finish().unwrap(); + refused( + &mut store, + GitBridgeEntry::Objects { + blobs: vec![map(x, da, a.len())], + trees: vec![GitTreeMapping { + git_oid: GitOid::compute_git_object("tree", b"not this tree"), + node: node.clone(), + }], + }, + &[(da, a)], + "a tree under a wrong Git ID", + ); + // The correct objects are accepted after those refusals. + let mut tree_payload = b"100644 a.txt\0".to_vec(); + tree_payload.extend_from_slice(&x.0); + let tree_oid = GitOid::compute_git_object("tree", &tree_payload); + let node_again = node.clone(); + store + .record_git_import( + &repo, + GitBridgeEntry::Objects { + blobs: vec![map(x, da, a.len())], + trees: vec![GitTreeMapping { + git_oid: tree_oid, + node, + }], + }, + &[(da, a)], + ) + .expect("the correct objects") + .expect("written"); + // A tree in another repository naming that blob, which no Git blob of that repository + // maps (export there would have no Git ID for it). + let other = RepoId::new("aien-dev/h1-other").unwrap(); + let before = store.committed_seq(); + match store.record_git_import( + &other, + GitBridgeEntry::Objects { + blobs: vec![], + trees: vec![GitTreeMapping { + git_oid: tree_oid, + node: node_again, + }], + }, + &[], + ) { + Err(e) => assert_eq!(e.code, RefusalCode::BlobNotFound, "{e}"), + Ok(s) => panic!("a tree naming a blob its repository does not map: accepted ({s:?})"), + } + assert_eq!(store.committed_seq(), before, "nothing written"); + let headers = b"author a 0 +0000\ncommitter a 0 +0000\n\nm\n".to_vec(); + let mut commit_payload = format!("tree {}\n", tree_oid.to_hex()).into_bytes(); + commit_payload.extend_from_slice(&headers); + let commit_oid = GitOid::compute_git_object("commit", &commit_payload); + let commit = |git_oid: GitOid| { + GitBridgeEntry::Commit(GitImportedCommit { + git_oid, + tree: tree_digest, + parents: vec![], + headers: headers.clone(), + }) + }; + let refused_after = |store: &mut SynapseStore, entry: GitBridgeEntry, case: &str| { + let before = store.committed_seq(); + match store.record_git_import(&repo, entry, &[]) { + Err(e) => assert_eq!(e.code, RefusalCode::GitObjectCorrupt, "{case}: {e}"), + Ok(s) => panic!("{case}: accepted ({s:?})"), + } + assert_eq!(store.committed_seq(), before, "{case}: nothing written"); + }; + refused_after( + &mut store, + commit(GitOid::compute_git_object("commit", b"another commit")), + "a commit under a wrong Git ID", + ); + store + .record_git_import(&repo, commit(commit_oid), &[]) + .expect("the correct commit") + .expect("written"); + let mut same_content = commit_oid; + same_content.0[0] ^= 1; + refused_after( + &mut store, + commit(same_content), + "a second Git ID for the same commit content", + ); + let seq = store.committed_seq(); + let tag = |git_oid: GitOid| { + GitBridgeEntry::Tag(synapse_store::git_bridge::GitImportedTag { + git_oid, + target: GitBridgeTarget::Blob(da), + rest: b"type blob\ntag v1\ntagger a 0 +0000\n\nt\n".to_vec(), + }) + }; + match store.record_git_import( + &repo, + tag(GitOid::compute_git_object("tag", b"another tag")), + &[], + ) { + Err(e) => assert_eq!(e.code, RefusalCode::GitObjectCorrupt, "{e}"), + Ok(s) => panic!("a tag under a wrong Git ID: accepted ({s:?})"), + } + assert_eq!(store.committed_seq(), seq, "nothing written"); + // After every refusal a full import into the same repository still works: export gives + // the fixture with identical IDs plus exactly the three objects mapped above. + let src = fixture("c2-poison-src"); + import_repository(&mut store, &repo, &src.join(".git"), ReadLimits::default()) + .expect("a correct import after refusals"); + let dest = root.join("export.git"); + export_store(&store, &repo, &BTreeMap::new(), &dest).expect("export"); + assert_fsck_strict_clean(&dest); + let mut expected = object_ids(&src); + expected.extend([x, tree_oid, commit_oid]); + assert_eq!(object_ids(&dest), expected); + assert_eq!(git_refs(&dest), git_refs(&src), "references"); + drop(store); + fs::remove_dir_all(&src).unwrap(); + fs::remove_dir_all(&root).unwrap(); +} + +/// Real repository round trip through the store (ignored by default: CI has no such +/// clone). `SYNAPSE_H1_REAL_REPO` names a `git clone --mirror` directory; run under +/// `/usr/bin/time -v` for peak RSS. +#[test] +#[ignore] +fn h1_real_repository_round_trips_through_the_store_with_identical_ids() { + let Ok(src) = std::env::var("SYNAPSE_H1_REAL_REPO") else { + panic!("set SYNAPSE_H1_REAL_REPO to a mirror clone's git directory"); + }; + let src = PathBuf::from(src); + let root = temp_dir("c2-real"); + let big = StoreLimits { + max_blob_bytes: 1 << 30, + max_total_changeset_bytes: 64 << 20, + ..limits() + }; + let mut store = open_store(&root.join("store"), big.clone()); + let started = std::time::Instant::now(); + let receipt = import(&mut store, &src).expect("import"); + let imported = started.elapsed(); + drop(store); + let started = std::time::Instant::now(); + let store = open_store(&root.join("store"), big); + let reopened = started.elapsed(); + let dest = root.join("export.git"); + let started = std::time::Instant::now(); + let pack = export_store(&store, &repo_id(), &BTreeMap::new(), &dest).expect("export"); + let exported = started.elapsed(); + assert_fsck_strict_clean(&dest); + assert_eq!(git_refs(&dest), git_refs(&src), "references"); + let reachable = |d: &Path| -> BTreeSet { + git_text(d, &["rev-list", "--objects", "--all"]) + .lines() + .map(|l| l[..40].to_string()) + .collect() + }; + let src_reachable = reachable(&src); + let dst_all: BTreeSet = object_ids(&dest).iter().map(|o| o.to_hex()).collect(); + assert_eq!(src_reachable, dst_all, "identical object IDs"); + println!( + "real repository {} through the store: {receipt:?}; import {:.1} s, reopen (WAL replay) {:.1} s, export {:.1} s, pack {} bytes", + src.display(), + imported.as_secs_f64(), + reopened.as_secs_f64(), + exported.as_secs_f64(), + pack.pack_bytes + ); + fs::remove_dir_all(&root).unwrap(); +} diff --git a/crates/synapse-git/tests/h1_git_bridge.rs b/crates/synapse-git/tests/h1_git_bridge.rs index 0c92dfd..6b25249 100644 --- a/crates/synapse-git/tests/h1_git_bridge.rs +++ b/crates/synapse-git/tests/h1_git_bridge.rs @@ -4,295 +4,16 @@ //! `verify-pack`, `fsck --full --strict`, `clone`). Nothing in `synapse-git` starts a Git //! process for these paths. -use std::collections::{BTreeMap, BTreeSet}; +mod common; + +use common::*; +use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -use std::process::{Command, Output}; +use std::process::Command; use synapse_core::{GitOid, RefusalCode}; use synapse_git::pack::{decode_idx, PackReader}; -use synapse_git::{export_repository, GitHistory, GitObject, GitRepository, ObjectKind}; - -fn temp_dir(label: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!( - "synapse-h1-{label}-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - fs::create_dir_all(&dir).unwrap(); - dir -} - -/// Run `git -C dir args` with no user or system configuration and fixed identities and dates, -/// so fixture history is the same on every host. -fn git_raw(dir: &Path, args: &[&str], stdin: Option<&[u8]>) -> Output { - let mut cmd = Command::new("git"); - cmd.arg("-C") - .arg(dir) - .args(args) - .env("GIT_CONFIG_NOSYSTEM", "1") - .env("GIT_CONFIG_GLOBAL", "/dev/null") - .env("GIT_AUTHOR_NAME", "Fixture Author") - .env("GIT_AUTHOR_EMAIL", "author@fixture.invalid") - .env("GIT_AUTHOR_DATE", "1700000000 +0000") - .env("GIT_COMMITTER_NAME", "Fixture Committer") - .env("GIT_COMMITTER_EMAIL", "committer@fixture.invalid") - .env("GIT_COMMITTER_DATE", "1700000000 +0000"); - if let Some(input) = stdin { - use std::io::Write; - cmd.stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); - let mut child = cmd.spawn().unwrap(); - child.stdin.take().unwrap().write_all(input).unwrap(); - child.wait_with_output().unwrap() - } else { - cmd.output().unwrap() - } -} - -fn git(dir: &Path, args: &[&str]) -> Vec { - let out = git_raw(dir, args, None); - assert!( - out.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - out.stdout -} - -fn git_text(dir: &Path, args: &[&str]) -> String { - String::from_utf8(git(dir, args)) - .unwrap() - .trim() - .to_string() -} - -fn write(dir: &Path, rel: &str, bytes: &[u8]) { - let path = dir.join(rel); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - fs::write(path, bytes).unwrap(); -} - -fn commit_all(dir: &Path, msg: &str) { - git(dir, &["add", "-A"]); - git(dir, &["commit", "-q", "--allow-empty", "-m", msg]); -} - -/// Deterministic pseudo-random bytes (xorshift), so the binary blob is incompressible. -fn noise(len: usize, seed: u64) -> Vec { - let mut x = seed | 1; - (0..len) - .map(|_| { - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - x as u8 - }) - .collect() -} - -/// Real Git history with every shape the bridge must carry: nested trees, executable files, -/// a symlink, an empty file, an incompressible binary, a large compressible file revised many -/// times (delta material), a two-parent merge, an octopus merge, an empty commit, annotated -/// tags of a commit, a tree, a blob and another tag, a lightweight tag, and a commit with a -/// multi-line signature header and an `encoding` header. -fn build_fixture(dir: &Path) { - git(dir, &["init", "-q", "-b", "main"]); - let mut big = String::new(); - for i in 0..6000 { - big.push_str(&format!( - "line {i}: the quick brown fox jumps over the lazy dog\n" - )); - } - write(dir, "README.md", b"# fixture\n"); - write(dir, "src/lib.rs", b"pub fn one() -> u32 { 1 }\n"); - write(dir, "src/deep/nested/mod.rs", b"// nested\n"); - write(dir, "data/empty.txt", b""); - write(dir, "data/noise.bin", &noise(40_000, 7)); - write(dir, "docs/big.txt", big.as_bytes()); - write(dir, "tools/run.sh", b"#!/bin/sh\necho run\n"); - let mut perms = fs::metadata(dir.join("tools/run.sh")) - .unwrap() - .permissions(); - std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755); - fs::set_permissions(dir.join("tools/run.sh"), perms).unwrap(); - std::os::unix::fs::symlink("../README.md", dir.join("docs/readme-link")).unwrap(); - commit_all(dir, "initial"); - for rev in 0..12 { - let edited = big.replace( - &format!("line {}:", rev * 400), - &format!("LINE {} (rev {rev}):", rev * 400), - ); - write(dir, "docs/big.txt", edited.as_bytes()); - write( - dir, - "src/lib.rs", - format!("pub fn one() -> u32 {{ {rev} }}\n").as_bytes(), - ); - commit_all(dir, &format!("revision {rev}")); - } - git(dir, &["tag", "-a", "v1", "-m", "release v1"]); - git(dir, &["tag", "light"]); - git(dir, &["checkout", "-q", "-b", "feature"]); - write(dir, "src/feature.rs", b"pub fn feature() {}\n"); - commit_all(dir, "feature work"); - git(dir, &["checkout", "-q", "main"]); - write(dir, "src/main_only.rs", b"fn main_only() {}\n"); - commit_all(dir, "main work"); - git( - dir, - &["merge", "-q", "--no-ff", "feature", "-m", "merge feature"], - ); - for name in ["oct-a", "oct-b"] { - git(dir, &["checkout", "-q", "-b", name, "main"]); - write(dir, &format!("{name}.txt"), name.as_bytes()); - commit_all(dir, name); - } - git(dir, &["checkout", "-q", "main"]); - git( - dir, - &["merge", "-q", "--no-ff", "oct-a", "oct-b", "-m", "octopus"], - ); - commit_all(dir, "empty commit"); - let tree = git_text(dir, &["rev-parse", "HEAD^{tree}"]); - let blob = git_text(dir, &["rev-parse", "HEAD:README.md"]); - git( - dir, - &["tag", "-a", "tree-tag", &tree, "-m", "a tag of a tree"], - ); - git( - dir, - &["tag", "-a", "blob-tag", &blob, "-m", "a tag of a blob"], - ); - git( - dir, - &[ - "tag", - "-a", - "-f", - "nested-tag", - "v1", - "-m", - "a tag of a tag", - ], - ); - // Objects reachable only through a tag: a blob in no tree, and an inner tag whose own - // reference is deleted, so only the outer tag reaches it. - let out = git_raw( - dir, - &["hash-object", "-w", "--stdin"], - Some(b"reachable only through a tag\n"), - ); - let lone_blob = String::from_utf8(out.stdout).unwrap().trim().to_string(); - git( - dir, - &[ - "tag", - "-a", - "lone-blob-tag", - &lone_blob, - "-m", - "tag of a lone blob", - ], - ); - git(dir, &["tag", "-a", "inner", "v1", "-m", "inner tag"]); - git( - dir, - &["tag", "-a", "-f", "outer-only", "inner", "-m", "outer tag"], - ); - git(dir, &["tag", "-d", "inner"]); - // A commit with a multi-line signature header and an encoding header, written as raw - // bytes (no signing key is involved: the header is opaque data to Git and to the bridge). - let head = git_text(dir, &["rev-parse", "HEAD"]); - let raw = format!( - "tree {tree}\nparent {head}\n\ -author Fixture Author 1700000001 +0100\n\ -committer Fixture Committer 1700000002 -0230\n\ -encoding ISO-8859-1\n\ -gpgsig -----BEGIN PGP SIGNATURE-----\n \n iQEzBAABCAAdFiEEfixture\n =abcd\n -----END PGP SIGNATURE-----\n\ -\nsigned commit with caf\u{e9}\n" - ); - let out = git_raw( - dir, - &["hash-object", "-t", "commit", "-w", "--stdin"], - Some(raw.as_bytes()), - ); - assert!(out.status.success()); - let signed = String::from_utf8(out.stdout).unwrap().trim().to_string(); - git(dir, &["update-ref", "refs/heads/signed", &signed]); -} - -/// `git cat-file --batch-all-objects --batch`: every object in the database as (kind, payload). -fn git_all_objects(dir: &Path) -> BTreeMap)> { - let out = git(dir, &["cat-file", "--batch-all-objects", "--batch"]); - let mut objects = BTreeMap::new(); - let mut rest = &out[..]; - while !rest.is_empty() { - let nl = rest.iter().position(|&b| b == b'\n').unwrap(); - let header = std::str::from_utf8(&rest[..nl]).unwrap(); - let mut parts = header.split(' '); - let oid = GitOid::from_hex(parts.next().unwrap()).unwrap(); - let kind = parts.next().unwrap().to_string(); - let size: usize = parts.next().unwrap().parse().unwrap(); - let payload = rest[nl + 1..nl + 1 + size].to_vec(); - objects.insert(oid, (kind, payload)); - rest = &rest[nl + 1 + size + 1..]; - } - objects -} - -fn git_refs(dir: &Path) -> BTreeMap { - git_text(dir, &["for-each-ref", "--format=%(refname) %(objectname)"]) - .lines() - .map(|l| { - let (name, oid) = l.split_once(' ').unwrap(); - (name.to_string(), GitOid::from_hex(oid).unwrap()) - }) - .collect() -} - -fn assert_history_matches_git(history: &GitHistory, dir: &Path) { - let oracle = git_all_objects(dir); - let native: BTreeSet = history.objects.keys().copied().collect(); - let expected: BTreeSet = oracle.keys().copied().collect(); - assert_eq!( - native, expected, - "native reachable set != git object database" - ); - for (oid, object) in &history.objects { - let (kind, payload) = &oracle[oid]; - assert_eq!(object.kind.name(), kind, "kind of {oid}"); - assert_eq!(&object.payload, payload, "payload of {oid}"); - assert_eq!(object.oid(), *oid); - } - assert_eq!(history.refs, git_refs(dir), "references"); -} - -fn assert_fsck_strict_clean(dir: &Path) { - let out = git_raw(dir, &["fsck", "--full", "--strict", "--no-dangling"], None); - let stderr = String::from_utf8_lossy(&out.stderr); - let stdout = String::from_utf8_lossy(&out.stdout); - assert!( - out.status.success() && !stderr.contains("error") && !stderr.contains("warning"), - "git fsck --full --strict failed in {}:\n{stdout}\n{stderr}", - dir.display() - ); -} - -/// Number of entries whose pack type (`gitformat-pack`: 3 bits after the MSB of the first byte) is `t`. -fn count_entry_type(pack: &[u8], entries: &[synapse_git::PackIndexEntry], t: u8) -> usize { - entries - .iter() - .filter(|e| (pack[e.offset as usize] >> 4) & 0x07 == t) - .count() -} - -fn objects_of(history: &GitHistory) -> Vec { - history.objects.values().cloned().collect() -} +use synapse_git::{export_repository, GitRepository, ObjectKind}; #[test] fn h1_native_reader_resolves_symbolic_ref_chains_and_loose_shadowing() { diff --git a/crates/synapse-git/tests/h1_hostile_repo.rs b/crates/synapse-git/tests/h1_hostile_repo.rs index b4a9b15..9a4fb86 100644 --- a/crates/synapse-git/tests/h1_hostile_repo.rs +++ b/crates/synapse-git/tests/h1_hostile_repo.rs @@ -173,6 +173,22 @@ fn h1_hostile_reference_linked_outside_the_repository_is_refused() { ); } +#[test] +fn h1_hostile_linked_object_fanout_directory_is_refused() { + // Re-review of PR #90, N1: `objects/ab` replaced by a link to a directory outside the + // repository that holds the very same object file. Following the link would read it. + let (root, git_dir, tip) = small_repo("fanout-link"); + let fanout = git_dir.join("objects").join(&tip[..2]); + let outside = root.join("outside-fanout"); + fs::rename(&fanout, &outside).unwrap(); + std::os::unix::fs::symlink(&outside, &fanout).unwrap(); + assert_refused( + read_history(&git_dir), + RefusalCode::PathInvalid, + "objects/xx -> directory outside", + ); +} + #[test] fn h1_hostile_oversized_reference_file_is_refused() { let (_root, git_dir, tip) = small_repo("big-ref"); diff --git a/crates/synapse-store/src/a1_tests.rs b/crates/synapse-store/src/a1_tests.rs index 735a56d..ac03f49 100644 --- a/crates/synapse-store/src/a1_tests.rs +++ b/crates/synapse-store/src/a1_tests.rs @@ -407,7 +407,7 @@ fn a1_documented_layout_decodes_and_every_non_canonical_form_is_refused() { Err(CodecRefusal::VersionUnsupported { found: 1 }) ); let mut k = review.clone(); - k[1] = 10; // first unused kind (7 and 8 are the B1 revision records, 9 the B4 state move) + k[1] = 12; // first unused kind (7, 8, 9 and 11 are revision records, 10 the H1 Git bridge) assert_eq!(kind(decode_record(&k, CAP)), "UnknownTag"); let mut k0 = review.clone(); k0[1] = 0; diff --git a/crates/synapse-store/src/codec.rs b/crates/synapse-store/src/codec.rs index d38ad08..d079d81 100644 --- a/crates/synapse-store/src/codec.rs +++ b/crates/synapse-store/src/codec.rs @@ -18,9 +18,13 @@ //! huge length is refused without allocating it. use super::*; +use crate::git_bridge::{ + GitBlobMapping, GitBridgeEntry, GitBridgeTarget, GitImportedCommit, GitImportedRefs, + GitImportedTag, GitTreeMapping, +}; use crate::seal::REVISION_SEAL_TAG; use synapse_auth::CapabilityLink; -use synapse_core::{FileMode, GitOid, PathMutation}; +use synapse_core::{FileMode, GitOid, PathMutation, TreeDigest}; /// Codec version byte that starts every binary record and checkpoint content. pub const CODEC_V2: u8 = 2; @@ -373,10 +377,102 @@ pub fn encode_record(record: &WalRecord) -> SynapseResult> { w.u64(*admitted_at_ms); put_seal(&mut w, seal); } + WalRecord::GitBridgeImported { + repo_id, + entry, + seal, + } => { + w.repo(repo_id)?; + put_git_bridge_entry(&mut w, entry)?; + put_seal(&mut w, seal); + } } Ok(w.0) } +/// The body of a kind 10 record without its seal trailer, `repo_id | entry`: what the +/// record digest (state root and seal chain) covers. +pub(crate) fn git_bridge_body(repo_id: &RepoId, entry: &GitBridgeEntry) -> SynapseResult> { + let mut w = Writer(Vec::with_capacity(256)); + w.repo(repo_id)?; + put_git_bridge_entry(&mut w, entry)?; + Ok(w.0) +} + +fn put_git_target(w: &mut Writer, t: &GitBridgeTarget) { + match t { + GitBridgeTarget::Commit(c) => { + w.u8(1); + w.raw(&c.0); + } + GitBridgeTarget::Tree(d) => { + w.u8(2); + w.raw(&d.0); + } + GitBridgeTarget::Blob(b) => { + w.u8(3); + w.raw(&b.0); + } + GitBridgeTarget::Tag(o) => { + w.u8(4); + w.raw(&o.0); + } + } +} + +/// `entry tag (u8) | body`: 1 objects `u32 n | (git oid [20] | blob [32] | size u64)* | u32 m +/// | (git oid [20] | u32 len | tree node)*`; 2 commit `git oid | tree [32] | u32 n | parent +/// [32]* | u32 len | headers`; 3 tag `git oid | target | u32 len | rest`; 4 refs `u32 n | +/// (name | target)* | option head`. A target is `tag (u8) | [32]` (1 commit, 2 tree, 3 blob) +/// or `4 | git oid [20]` (tag). +fn put_git_bridge_entry(w: &mut Writer, e: &GitBridgeEntry) -> SynapseResult<()> { + w.u8(e.tag()); + match e { + GitBridgeEntry::Objects { blobs, trees } => { + w.len(blobs.len(), "Git blob count")?; + for b in blobs { + w.raw(&b.git_oid.0); + w.raw(&b.blob.0); + w.u64(b.size_bytes); + } + w.len(trees.len(), "Git tree count")?; + for t in trees { + w.raw(&t.git_oid.0); + w.bytes(&t.node, "Git tree node")?; + } + } + GitBridgeEntry::Commit(c) => { + w.raw(&c.git_oid.0); + w.raw(&c.tree.0); + w.len(c.parents.len(), "Git parent count")?; + for p in &c.parents { + w.raw(&p.0); + } + w.bytes(&c.headers, "Git commit headers")?; + } + GitBridgeEntry::Tag(t) => { + w.raw(&t.git_oid.0); + put_git_target(w, &t.target); + w.bytes(&t.rest, "Git tag body")?; + } + GitBridgeEntry::Refs(r) => { + w.len(r.refs.len(), "Git reference count")?; + for (name, target) in &r.refs { + w.str(name, "Git reference name")?; + put_git_target(w, target); + } + match &r.head { + Some(h) => { + w.bool(true); + w.str(h, "Git HEAD")?; + } + None => w.bool(false), + } + } + } + Ok(()) +} + /// The optional seal trailer of a revision record (issue #101): nothing when unsealed, /// else `1 u8 | server_key [32] | signature [64]`. Bytes after a body that do not open /// with the seal tag are refused as `TrailingBytes`. @@ -737,6 +833,93 @@ fn get_version(r: &mut Reader) -> Decoded<()> { } /// Decode a binary v2 record. `cap` bounds every declared length and every reservation. +const MIN_GIT_TARGET: usize = 1 + 20; + +fn get_git_target(r: &mut Reader) -> Decoded { + let at = r.pos; + Ok(match r.u8()? { + 1 => GitBridgeTarget::Commit(ChangesetId(r.array()?)), + 2 => GitBridgeTarget::Tree(TreeDigest(r.array()?)), + 3 => GitBridgeTarget::Blob(BlobDigest(r.array()?)), + 4 => GitBridgeTarget::Tag(GitOid(r.array()?)), + tag => { + return Err(CodecRefusal::UnknownTag { + at, + field: "Git bridge target", + tag, + }) + } + }) +} + +fn get_git_bridge_entry(r: &mut Reader) -> Decoded { + let at = r.pos; + Ok(match r.u8()? { + 1 => { + let (n, mut blobs) = r.vec(20 + 32 + 8)?; + for _ in 0..n { + blobs.push(GitBlobMapping { + git_oid: GitOid(r.array()?), + blob: BlobDigest(r.array()?), + size_bytes: r.u64()?, + }); + } + let (m, mut trees) = r.vec(20 + 4 + 4)?; + for _ in 0..m { + let git_oid = GitOid(r.array()?); + let node = r.embedded("Git tree node", |b| { + synapse_core::decode_tree_node(b).map(|_| b.to_vec()) + })?; + trees.push(GitTreeMapping { git_oid, node }); + } + GitBridgeEntry::Objects { blobs, trees } + } + 2 => { + let git_oid = GitOid(r.array()?); + let tree = TreeDigest(r.array()?); + let (n, mut parents) = r.vec(32)?; + for _ in 0..n { + parents.push(ChangesetId(r.array()?)); + } + let headers = r.bytes()?; + GitBridgeEntry::Commit(GitImportedCommit { + git_oid, + tree, + parents, + headers, + }) + } + 3 => GitBridgeEntry::Tag(GitImportedTag { + git_oid: GitOid(r.array()?), + target: get_git_target(r)?, + rest: r.bytes()?, + }), + 4 => { + let (n, mut refs) = r.vec::<(String, GitBridgeTarget)>(MIN_STR + MIN_GIT_TARGET)?; + for _ in 0..n { + let name_at = r.pos; + let name = r.string()?; + if refs.last().is_some_and(|(p, _)| *p >= name) { + return Err(CodecRefusal::NotCanonical { + at: name_at, + field: "Git reference names", + }); + } + refs.push((name, get_git_target(r)?)); + } + let head = if r.bool()? { Some(r.string()?) } else { None }; + GitBridgeEntry::Refs(GitImportedRefs { refs, head }) + } + tag => { + return Err(CodecRefusal::UnknownTag { + at, + field: "Git bridge entry", + tag, + }) + } + }) +} + pub fn decode_record(bytes: &[u8], cap: u64) -> Result { let mut r = Reader::new(bytes, cap); get_version(&mut r)?; @@ -828,6 +1011,15 @@ pub fn decode_record(bytes: &[u8], cap: u64) -> Result admitted_at_ms: r.u64()?, seal: get_seal(&mut r)?, }, + 10 => { + let repo_id = r.repo()?; + let entry = get_git_bridge_entry(&mut r)?; + WalRecord::GitBridgeImported { + repo_id, + entry, + seal: get_seal(&mut r)?, + } + } tag => { return Err(CodecRefusal::UnknownTag { at, @@ -899,7 +1091,7 @@ pub struct CheckpointState { pub blob_first_visible_seq: BTreeMap, pub github_projections: BTreeMap, pub canonical_heads: BTreeMap, - /// Revision records (kinds 7, 8, and 9) in sequence order (ADR 0003 B1, B4). Written only when + /// Revision records (kinds 7 to 11; 10 is the H1 Git bridge) in sequence order (ADR 0003 B1, B4, B5, H1). Written only when /// non-empty, so checkpoints of stores without revisions keep their exact bytes. pub revision_log: Vec<(WalSeq, WalRecord)>, } @@ -1186,7 +1378,7 @@ impl CheckpointState { let tag_at = r.pos + 4 + 1; let bytes = r.bytes()?; let record = decode_record(&bytes, r.cap)?; - if !matches!(record.kind_byte(), 7..=9 | 11) { + if !matches!(record.kind_byte(), 7..=11) { return Err(CodecRefusal::UnknownTag { at: tag_at, field: "revision record kind", diff --git a/crates/synapse-store/src/git_bridge.rs b/crates/synapse-store/src/git_bridge.rs new file mode 100644 index 0000000..b89a54c --- /dev/null +++ b/crates/synapse-store/src/git_bridge.rs @@ -0,0 +1,1019 @@ +//! Git bridge records and the persistent `GitOid` map (ADR 0003 section 5.10, step H1 cut 2). +//! See `docs/H1_GIT_BRIDGE.md`. +//! +//! An import is a sequence of WAL kind 10 records ([`crate::WalRecord::GitBridgeImported`]), +//! one [`GitBridgeEntry`] each: +//! +//! * `Objects`: Git blobs mapped to `BlobDigest`s (their bytes stored as ordinary blobs) and Git +//! trees mapped to canonical `TreeDigest` nodes (C1). A Git tree ID is never stored as native +//! identity: at export it is recomputed from the node. +//! * `Commit`: an imported revision. Its native form is the commit's `TreeDigest`, its parents +//! as imported revision IDs in Git's written order, and the rest of the commit (author, +//! committer, extra headers such as `gpgsig` and `mergetag`, message) as opaque bytes. Its +//! `ChangesetId` is [`GitImportedCommit::changeset_id`]. +//! * `Tag`: an annotated tag, its target as a native identity and every byte after the +//! `object` line opaque. +//! * `Refs`: the repository's references and `HEAD` at import. +//! +//! The ledger is a pure function of the kind 10 records that took effect, in `WalSeq` order: +//! replay, a checkpoint (which keeps the records) and a live store rebuild the same map. In a +//! store opened with a seal key, kind 10 is a revision record like kinds 7, 8, 9 and 11: it is +//! written sealed and applied only when its seal verifies against the chain so far (issue +//! #101). Every check runs before the write ([`crate::SynapseStore::record_git_import`]); +//! apply itself is total. + +use crate::WalRecord; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use synapse_core::{ + decode_tree_node, git_tree_payload_of_node, BlobDigest, ChangesetId, GitOid, RefusalCode, + RepoId, SynapseError, SynapseResult, TreeDigest, TreeNodeEntryKind, WalSeq, +}; + +/// Domain tag of an imported revision's `ChangesetId`. Distinct from the v1 and v2 domains, so +/// an imported revision never shares an identity with an agent revision. +pub const DOMAIN_GIT_IMPORTED_REVISION_V1: &[u8] = b"SYNAPSE_GIT_IMPORTED_REVISION_V1\0"; + +/// Version of the imported revision encoding hashed into its `ChangesetId`. +pub const GIT_IMPORTED_REVISION_VERSION: u16 = 1; + +/// Longest reference name the bridge stores. +pub const MAX_GIT_REF_NAME_BYTES: usize = 1024; + +/// One Git blob and the native blob holding its bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitBlobMapping { + pub git_oid: GitOid, + pub blob: BlobDigest, + pub size_bytes: u64, +} + +/// One Git tree and its canonical native node (C1 version 1 encoding). The `TreeDigest` is +/// derived from the node bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitTreeMapping { + pub git_oid: GitOid, + pub node: Vec, +} + +impl GitTreeMapping { + pub fn digest(&self) -> TreeDigest { + TreeDigest::of_encoded_node(&self.node) + } +} + +/// An imported Git commit: tree, parents in written order, and the opaque remainder of the +/// commit object (every byte after the `tree` and `parent` lines). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitImportedCommit { + /// The original Git commit ID (the map key; checked again at every export). + pub git_oid: GitOid, + pub tree: TreeDigest, + /// Imported revision IDs of the parents, in the commit's written order. + pub parents: Vec, + /// Author, committer, extra headers and message, byte for byte. + pub headers: Vec, +} + +impl GitImportedCommit { + /// `version u16 (1) | tree [32] | u32 n | parent [32]*n | u32 len | headers`. + pub fn encode_content(&self) -> Vec { + let mut buf = Vec::with_capacity(46 + 32 * self.parents.len() + self.headers.len()); + buf.extend_from_slice(&GIT_IMPORTED_REVISION_VERSION.to_le_bytes()); + buf.extend_from_slice(&self.tree.0); + buf.extend_from_slice(&(self.parents.len() as u32).to_le_bytes()); + for p in &self.parents { + buf.extend_from_slice(&p.0); + } + buf.extend_from_slice(&(self.headers.len() as u32).to_le_bytes()); + buf.extend_from_slice(&self.headers); + buf + } + + /// `SHA-256(DOMAIN_GIT_IMPORTED_REVISION_V1 || u64 len || encode_content())`. It covers the + /// tree, the parents in order and the opaque headers, so two Git commits are one imported + /// revision exactly when their objects are identical; the Git ID itself is not hashed in. + pub fn changeset_id(&self) -> ChangesetId { + let bytes = self.encode_content(); + let mut h = Sha256::new(); + h.update(DOMAIN_GIT_IMPORTED_REVISION_V1); + h.update((bytes.len() as u64).to_le_bytes()); + h.update(&bytes); + ChangesetId(h.finalize().into()) + } +} + +/// A native identity a tag or reference points at. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum GitBridgeTarget { + /// An imported revision. + Commit(ChangesetId), + Tree(TreeDigest), + Blob(BlobDigest), + /// Another imported tag, by its Git ID (tags have no native identity). + Tag(GitOid), +} + +/// An imported annotated tag: its target and every byte after its `object` line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitImportedTag { + pub git_oid: GitOid, + pub target: GitBridgeTarget, + /// From the `type` line to the end of the object, byte for byte. + pub rest: Vec, +} + +/// The references of an imported repository: names strictly ascending, and the branch `HEAD` +/// names. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitImportedRefs { + pub refs: Vec<(String, GitBridgeTarget)>, + pub head: Option, +} + +/// The body of one kind 10 record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GitBridgeEntry { + Objects { + blobs: Vec, + trees: Vec, + }, + Commit(GitImportedCommit), + Tag(GitImportedTag), + Refs(GitImportedRefs), +} + +impl GitBridgeEntry { + /// Entry tag in the record encoding. + pub fn tag(&self) -> u8 { + match self { + Self::Objects { .. } => 1, + Self::Commit(_) => 2, + Self::Tag(_) => 3, + Self::Refs(_) => 4, + } + } + + /// Blob digests an `Objects` entry maps (all must be visible once the record commits). + pub fn blobs(&self) -> Vec { + match self { + Self::Objects { blobs, .. } => blobs.iter().map(|b| b.blob).collect(), + _ => Vec::new(), + } + } +} + +/// Is `name` a reference name the bridge accepts: under `refs/`, no empty, `.`-leading or +/// `..` components, no `.lock` suffix, no control or `" ~^:?*[\"` bytes, at most +/// [`MAX_GIT_REF_NAME_BYTES`]. Export writes each reference as a file below the new +/// repository, so this rule also keeps every write inside it. +pub fn valid_git_ref_name(name: &str) -> bool { + name.len() <= MAX_GIT_REF_NAME_BYTES + && name.starts_with("refs/") + && !name.ends_with('/') + && !name.ends_with(".lock") + && !name.contains("@{") + && name.split('/').all(|c| { + !c.is_empty() && !c.starts_with('.') && !c.ends_with(".lock") && !c.contains("..") + }) + && !name + .bytes() + .any(|b| b < 0x20 || b == 0x7f || b" ~^:?*[\\".contains(&b)) +} + +/// Blobs that come with one kind 10 record: digest and bytes. +pub type GitImportBlobs<'a> = Vec<(BlobDigest, &'a [u8])>; + +/// One record of a Git import batch: the entry and the blobs it brings. +pub type GitImportRecord<'a> = (GitBridgeEntry, GitImportBlobs<'a>); + +fn refused(code: RefusalCode, msg: impl Into) -> SynapseError { + SynapseError::new(code, msg) +} + +/// The persistent map of one repository. +#[derive(Debug, Clone, Default)] +pub struct GitBridgeRepo { + blobs: BTreeMap, + trees: BTreeMap, + commits: BTreeMap, + by_changeset: BTreeMap, + tags: BTreeMap, + refs: Option, + /// Reverse maps (native identity to Git ID), so a tree, commit or tag can be re-hashed. + blob_oids: BTreeMap, + tree_oids: BTreeMap, +} + +impl GitBridgeRepo { + /// The map: an imported commit's (`ChangesetId`, `TreeDigest`). + pub fn commit(&self, git_oid: GitOid) -> Option<(ChangesetId, TreeDigest)> { + self.commits.get(&git_oid).map(|(id, c)| (*id, c.tree)) + } + + /// The imported commit behind an imported revision ID. + pub fn commit_by_changeset(&self, id: ChangesetId) -> Option<&GitImportedCommit> { + self.by_changeset + .get(&id) + .and_then(|oid| self.commits.get(oid)) + .map(|(_, c)| c) + } + + pub fn blob(&self, git_oid: GitOid) -> Option<(BlobDigest, u64)> { + self.blobs.get(&git_oid).copied() + } + + pub fn tree(&self, git_oid: GitOid) -> Option { + self.trees.get(&git_oid).copied() + } + + pub fn tag(&self, git_oid: GitOid) -> Option<&GitImportedTag> { + self.tags.get(&git_oid) + } + + /// Every mapped blob: Git ID to (`BlobDigest`, size). + pub fn blobs(&self) -> &BTreeMap { + &self.blobs + } + + /// Every mapped tree: Git ID to `TreeDigest`. + pub fn trees(&self) -> &BTreeMap { + &self.trees + } + + /// Every imported commit by Git ID, with its imported revision ID. + pub fn commits(&self) -> impl Iterator { + self.commits.iter() + } + + pub fn tags(&self) -> &BTreeMap { + &self.tags + } + + /// The references of the latest `Refs` record. + pub fn refs(&self) -> Option<&GitImportedRefs> { + self.refs.as_ref() + } + + /// Commits, trees, blobs and tags held. + pub fn object_count(&self) -> usize { + self.blobs.len() + self.trees.len() + self.commits.len() + self.tags.len() + } +} + +/// In-memory bridge state, rebuilt from kind 10 records. +#[derive(Debug, Clone, Default)] +pub(crate) struct GitBridgeLedger { + repos: BTreeMap, + /// Canonical tree nodes by digest (content addressed, shared by every repository). + nodes: BTreeMap>, + /// Size of every blob a Git blob maps to (content addressed, shared). + blob_sizes: BTreeMap, +} + +impl GitBridgeLedger { + pub(crate) fn from_log<'a>(log: impl IntoIterator) -> Self { + let mut l = Self::default(); + for (_, record) in log { + l.apply(record); + } + l + } + + pub(crate) fn repo(&self, repo: &RepoId) -> Option<&GitBridgeRepo> { + self.repos.get(repo) + } + + pub(crate) fn node(&self, digest: TreeDigest) -> Option<&[u8]> { + self.nodes.get(&digest).map(Vec::as_slice) + } + + /// Every blob any repository maps (compaction keeps them). + pub(crate) fn blob_digests(&self) -> impl Iterator + '_ { + self.blob_sizes.keys().copied() + } + + /// Apply one kind 10 record (other records are ignored). Total: replay applies every + /// record the same way on every replica. + pub(crate) fn apply(&mut self, record: &WalRecord) { + let WalRecord::GitBridgeImported { repo_id, entry, .. } = record else { + return; + }; + let repo = self.repos.entry(repo_id.clone()).or_default(); + match entry { + GitBridgeEntry::Objects { blobs, trees } => { + for b in blobs { + repo.blobs.insert(b.git_oid, (b.blob, b.size_bytes)); + repo.blob_oids.insert(b.blob, b.git_oid); + self.blob_sizes.entry(b.blob).or_insert(b.size_bytes); + } + for t in trees { + let d = t.digest(); + repo.trees.insert(t.git_oid, d); + repo.tree_oids.insert(d, t.git_oid); + self.nodes.entry(d).or_insert_with(|| t.node.clone()); + } + } + GitBridgeEntry::Commit(c) => { + let id = c.changeset_id(); + repo.by_changeset.insert(id, c.git_oid); + repo.commits.insert(c.git_oid, (id, c.clone())); + } + GitBridgeEntry::Tag(t) => { + repo.tags.insert(t.git_oid, t.clone()); + } + GitBridgeEntry::Refs(r) => repo.refs = Some(r.clone()), + } + } + + /// Check one entry against the map before it is written. Returns `false` when the entry + /// is already fully recorded (nothing to write). `provided` holds the blobs whose bytes + /// come with the record; `blob_visible` says whether a blob is already committed and + /// `read_blob` reads one that is. + /// + /// Beyond structure (sizes, visibility, subtrees, parents and targets imported in this + /// repository, no Git ID remapped), every Git ID the entry newly maps is re-hashed from + /// the content it maps to (blob bytes; tree, commit and tag payloads rebuilt from the + /// stored form and the Git IDs already mapped), and a Git ID may appear only once in an + /// entry. So a record cannot map a Git ID to anything export would not give back. + pub(crate) fn check( + &self, + repo_id: &RepoId, + entry: &GitBridgeEntry, + provided: &BTreeMap, + blob_visible: &dyn Fn(&BlobDigest) -> bool, + read_blob: &dyn Fn(&BlobDigest) -> SynapseResult>, + ) -> SynapseResult { + let repo = self.repos.get(repo_id); + let conflict = |what: &str, oid: GitOid| { + refused( + RefusalCode::GitObjectCorrupt, + format!("{what} {oid} is already mapped to a different native object"), + ) + }; + match entry { + GitBridgeEntry::Objects { blobs, trees } => { + if blobs.is_empty() && trees.is_empty() { + return Err(refused( + RefusalCode::GitObjectCorrupt, + "an objects entry maps no object", + )); + } + let mut ids: BTreeSet = BTreeSet::new(); + for oid in blobs + .iter() + .map(|b| b.git_oid) + .chain(trees.iter().map(|t| t.git_oid)) + { + if !ids.insert(oid) { + return Err(refused( + RefusalCode::GitObjectCorrupt, + format!("Git ID {oid} appears more than once in one objects entry"), + )); + } + } + let mut fresh = false; + // Blobs this entry maps: digest to (Git ID, size). + let mut entry_blobs: BTreeMap = BTreeMap::new(); + for b in blobs { + let known = match repo.and_then(|r| r.blobs.get(&b.git_oid)) { + Some(&(d, s)) if d == b.blob && s == b.size_bytes => true, + Some(_) => return Err(conflict("Git blob", b.git_oid)), + None => false, + }; + match provided.get(&b.blob) { + Some(bytes) if bytes.len() as u64 != b.size_bytes => { + return Err(refused( + RefusalCode::BlobDigestMismatch, + format!( + "Git blob {} declares {} bytes but its blob holds {}", + b.git_oid, + b.size_bytes, + bytes.len() + ), + )) + } + Some(_) => {} + None if blob_visible(&b.blob) => {} + None => { + return Err(refused( + RefusalCode::BlobNotFound, + format!("Git blob {} maps to missing blob {}", b.git_oid, b.blob), + )) + } + } + if !known { + fresh = true; + let read; + let content: &[u8] = match provided.get(&b.blob) { + Some(bytes) => bytes, + None => { + read = read_blob(&b.blob)?; + &read + } + }; + if content.len() as u64 != b.size_bytes { + return Err(refused( + RefusalCode::BlobDigestMismatch, + format!( + "Git blob {} declares {} bytes but its blob holds {}", + b.git_oid, + b.size_bytes, + content.len() + ), + )); + } + verify_git_id("blob", b.git_oid, content)?; + } + entry_blobs.insert(b.blob, (b.git_oid, b.size_bytes)); + } + let mapped_blob = |blob: &BlobDigest| -> Option<(GitOid, u64)> { + entry_blobs.get(blob).copied().or_else(|| { + let r = repo?; + let oid = *r.blob_oids.get(blob)?; + r.blobs.get(&oid).map(|&(_, s)| (oid, s)) + }) + }; + // Trees this entry maps: digest to Git ID. + let mut entry_trees: BTreeMap = BTreeMap::new(); + for t in trees { + let digest = t.digest(); + let known = match repo.and_then(|r| r.trees.get(&t.git_oid)) { + Some(&d) if d == digest => true, + Some(_) => return Err(conflict("Git tree", t.git_oid)), + None => false, + }; + let mapped_tree = |child: &TreeDigest| -> Option { + entry_trees + .get(child) + .or_else(|| repo.and_then(|r| r.tree_oids.get(child))) + .copied() + }; + for e in decode_tree_node(&t.node)? { + match e.kind { + TreeNodeEntryKind::File { + blob, size_bytes, .. + } => match mapped_blob(&blob) { + None => { + return Err(refused( + RefusalCode::BlobNotFound, + format!( + "tree {} names blob {blob} that no Git blob of this repository maps", + t.git_oid + ), + )) + } + Some((_, s)) if s != size_bytes => { + return Err(refused( + RefusalCode::BlobDigestMismatch, + format!( + "tree {} gives {} the size {size_bytes} but its blob holds {s}", + t.git_oid, e.name + ), + )) + } + Some(_) => {} + }, + TreeNodeEntryKind::Dir(child) => { + if child == TreeDigest::empty() || mapped_tree(&child).is_none() { + return Err(refused( + RefusalCode::TreeNodeNotFound, + format!( + "tree {} names subtree {child} that is empty or not imported", + t.git_oid + ), + )); + } + } + } + } + if !known { + fresh = true; + let unmapped = || refused(RefusalCode::GitObjectCorrupt, "unmapped entry"); + let payload = git_tree_payload_of_node( + &t.node, + &mut |b| mapped_blob(&b).map(|(o, _)| o).ok_or_else(unmapped), + &mut |c| mapped_tree(&c).ok_or_else(unmapped), + )?; + verify_git_id("tree", t.git_oid, &payload)?; + } + entry_trees.insert(digest, t.git_oid); + } + Ok(fresh) + } + GitBridgeEntry::Commit(c) => { + let id = c.changeset_id(); + if let Some((existing, _)) = repo.and_then(|r| r.commits.get(&c.git_oid)) { + return if *existing == id { + Ok(false) + } else { + Err(conflict("Git commit", c.git_oid)) + }; + } + if let Some(other) = repo.and_then(|r| r.by_changeset.get(&id)) { + return Err(refused( + RefusalCode::GitObjectCorrupt, + format!( + "commit {} has the same content as the imported commit {other}", + c.git_oid + ), + )); + } + let tree = target_oid(repo, &GitBridgeTarget::Tree(c.tree)).map_err(|_| { + refused( + RefusalCode::TreeNodeNotFound, + format!( + "commit {} names tree {} that is not imported", + c.git_oid, c.tree + ), + ) + })?; + let mut payload = format!("tree {}\n", tree.to_hex()).into_bytes(); + for p in &c.parents { + let Some(parent) = repo.and_then(|r| r.by_changeset.get(p)) else { + return Err(refused( + RefusalCode::ParentChangesetNotFound, + format!("commit {} names parent {p} that is not imported", c.git_oid), + )); + }; + payload.extend_from_slice(format!("parent {}\n", parent.to_hex()).as_bytes()); + } + payload.extend_from_slice(&c.headers); + verify_git_id("commit", c.git_oid, &payload)?; + Ok(true) + } + GitBridgeEntry::Tag(t) => { + if let Some(existing) = repo.and_then(|r| r.tags.get(&t.git_oid)) { + return if existing == t { + Ok(false) + } else { + Err(conflict("Git tag", t.git_oid)) + }; + } + let target = target_oid(repo, &t.target)?; + let mut payload = format!("object {}\n", target.to_hex()).into_bytes(); + payload.extend_from_slice(&t.rest); + verify_git_id("tag", t.git_oid, &payload)?; + Ok(true) + } + GitBridgeEntry::Refs(r) => { + check_git_ref_names(r.refs.iter().map(|(n, _)| n.as_str()), r.head.as_deref())?; + for (_, target) in &r.refs { + target_oid(repo, target)?; + } + Ok(repo.and_then(|x| x.refs.as_ref()) != Some(r)) + } + } + } +} + +/// The Git ID a bridge target names in `repo`, refused unless it is imported there (the +/// empty tree is always known: its Git ID is that of the empty Git tree). +fn target_oid(repo: Option<&GitBridgeRepo>, target: &GitBridgeTarget) -> SynapseResult { + let oid = match target { + GitBridgeTarget::Commit(id) => repo.and_then(|r| r.by_changeset.get(id).copied()), + GitBridgeTarget::Tree(d) if *d == TreeDigest::empty() => { + Some(GitOid::compute_git_object("tree", b"")) + } + GitBridgeTarget::Tree(d) => repo.and_then(|r| r.tree_oids.get(d).copied()), + GitBridgeTarget::Blob(b) => repo.and_then(|r| r.blob_oids.get(b).copied()), + GitBridgeTarget::Tag(oid) => repo.filter(|r| r.tags.contains_key(oid)).map(|_| *oid), + }; + oid.ok_or_else(|| { + refused( + RefusalCode::GitObjectCorrupt, + format!("bridge target {target:?} is not imported"), + ) + }) +} + +/// Refuse a Git ID that is not the SHA-1 of its object (`{kind} {len}\0{payload}`). +fn verify_git_id(kind: &str, claimed: GitOid, payload: &[u8]) -> SynapseResult<()> { + let actual = GitOid::compute_git_object(kind, payload); + if actual == claimed { + Ok(()) + } else { + Err(refused( + RefusalCode::GitObjectCorrupt, + format!("Git {kind} {claimed} does not match its content (which hashes to {actual})"), + )) + } +} + +/// Check reference names as a `Refs` record must hold them: each [`valid_git_ref_name`], +/// strictly ascending, no name a directory prefix of another (`refs/a` beside `refs/a/b`: +/// one would be a file and the other a directory), and a valid `HEAD` target. +pub fn check_git_ref_names<'a>( + names: impl IntoIterator, + head: Option<&str>, +) -> SynapseResult<()> { + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for name in names { + if !valid_git_ref_name(name) { + return Err(refused( + RefusalCode::PathInvalid, + format!("invalid reference name {name:?}"), + )); + } + if seen.last().is_some_and(|p| *p >= name) { + return Err(refused( + RefusalCode::PathInvalid, + "reference names must be strictly ascending", + )); + } + // A prefix sorts first, so it is already in `seen`. + for (i, _) in name.match_indices('/') { + if seen.contains(&name[..i]) { + return Err(refused( + RefusalCode::PathInvalid, + format!("reference {} is a prefix directory of {name}", &name[..i]), + )); + } + } + seen.insert(name); + } + if let Some(head) = head { + if !valid_git_ref_name(head) { + return Err(refused( + RefusalCode::PathInvalid, + format!("invalid HEAD target {head:?}"), + )); + } + } + Ok(()) +} + +/// Digest input of one kind 10 record for the state root and the seal chain: its body +/// (`repo_id | entry`) without the seal trailer, length first. +pub(crate) fn record_digest_input(repo_id: &RepoId, entry: &GitBridgeEntry, h: &mut Sha256) { + if let Ok(bytes) = crate::codec::git_bridge_body(repo_id, entry) { + h.update((bytes.len() as u64).to_le_bytes()); + h.update(&bytes); + } +} + +impl crate::SynapseStore { + /// Write one Git bridge record (WAL kind 10) for `repo_id`: a batch of one, see + /// [`Self::record_git_imports`]. Returns `None` when the entry is already recorded. + pub fn record_git_import( + &mut self, + repo_id: &RepoId, + entry: GitBridgeEntry, + blobs: &[(BlobDigest, &[u8])], + ) -> SynapseResult> { + let mut out = self.record_git_imports(repo_id, vec![(entry, blobs.to_vec())])?; + Ok(out.pop().flatten()) + } + + /// Write a whole Git import (WAL kind 10 records, in order) for `repo_id`, checked as one + /// unit before the first write. Every record of a batch of more than one is first run + /// against a copy of the map in which the earlier records of the batch are already applied: binary codec, blob caps, + /// each provided blob's digest and that the entry maps it, [`GitBridgeLedger::check`] + /// (structure and every new Git ID re-hashed from its content) and the exact size of its + /// WAL frame against `max_wal_record_bytes`. Only when every record passes are they + /// written, each checked again against the live map. So any refusal the checks can give + /// leaves the store untouched; a failure while writing (an I/O error) can still stop a + /// batch part way, and a re-import then finishes it. Blobs not yet visible go inline. + /// Returns, per record, its sequence number or `None` when it was already recorded. + pub fn record_git_imports( + &mut self, + repo_id: &RepoId, + records: Vec>, + ) -> SynapseResult>> { + self.check_writable()?; + self.check_binary_codec()?; + // Dry run on a copy of the map. A single record needs none: its write checks every + // limit, the frame size included, before anything is written. + let mut sim = (records.len() > 1).then(|| self.git_bridge.clone()); + let mut sim_visible: BTreeSet = BTreeSet::new(); + let mut prepared = Vec::with_capacity(records.len()); + for (entry, blobs) in records { + let record = WalRecord::GitBridgeImported { + repo_id: repo_id.clone(), + entry, + seal: None, + }; + if let Some(sim) = sim.as_mut() { + let visible = |d: &BlobDigest| { + self.blob_first_visible_seq.contains_key(d) || sim_visible.contains(d) + }; + if let Some(inline) = self.prepare_git_import(sim, &record, &blobs, &visible)? { + self.check_frame_cap(self.git_import_frame_len(&record, &inline)?)?; + sim_visible.extend(inline.iter().map(|(d, _)| *d)); + sim.apply(&record); + } + } + prepared.push((record, blobs)); + } + drop(sim); + let mut out = Vec::with_capacity(prepared.len()); + for (record, blobs) in prepared { + let visible = |d: &BlobDigest| self.blob_first_visible_seq.contains_key(d); + let seq = match self.prepare_git_import(&self.git_bridge, &record, &blobs, &visible)? { + None => None, + Some(inline) => Some(self.append_revision_record(record, &inline, None)?), + }; + out.push(seq); + } + Ok(out) + } + + /// Every check of one kind 10 record against `ledger`. Returns the blobs to put inline, + /// or `None` when the entry is already recorded. + fn prepare_git_import<'b>( + &self, + ledger: &GitBridgeLedger, + record: &WalRecord, + blobs: &[(BlobDigest, &'b [u8])], + visible: &dyn Fn(&BlobDigest) -> bool, + ) -> SynapseResult>> { + let WalRecord::GitBridgeImported { repo_id, entry, .. } = record else { + return Err(refused( + RefusalCode::GitObjectCorrupt, + "not a Git bridge record", + )); + }; + let mapped: BTreeSet = entry.blobs().into_iter().collect(); + let mut provided: BTreeMap = BTreeMap::new(); + let mut total = 0u64; + for (declared, bytes) in blobs { + let len = bytes.len() as u64; + total = total.saturating_add(len); + if len > self.limits.max_blob_bytes || total > self.limits.max_total_changeset_bytes { + return Err(refused( + RefusalCode::QuotaPayloadTooLarge, + "Git bridge blobs exceed max_blob_bytes or max_total_changeset_bytes", + )); + } + let actual = BlobDigest::compute(bytes); + if actual != *declared { + return Err(refused( + RefusalCode::BlobDigestMismatch, + format!("declared blob digest {declared} does not match computed {actual}"), + )); + } + if !mapped.contains(declared) { + return Err(refused( + RefusalCode::GitObjectCorrupt, + format!("blob {declared} is provided but no Git blob maps to it"), + )); + } + provided.insert(*declared, bytes); + } + let read_blob = |d: &BlobDigest| self.get_blob_visible(*d, None, None); + if !ledger.check(repo_id, entry, &provided, visible, &read_blob)? { + return Ok(None); + } + let mut inline: GitImportBlobs<'b> = Vec::new(); + for (digest, bytes) in blobs { + if !visible(digest) && !inline.iter().any(|(d, _)| d == digest) { + inline.push((*digest, *bytes)); + } + } + Ok(Some(inline)) + } + + /// The exact WAL payload length `append_revision_record` gives `record` with `inline` + /// blobs (v2: `u32 meta_len | meta | u32 count | (digest | u32 len | bytes)*`; v1: the + /// metadata, blobs go to files). A sealed store adds the seal trailer `tag | server key | + /// signature` (1 + 32 + 64 bytes) that the write seals `record` with. + fn git_import_frame_len( + &self, + record: &WalRecord, + inline: &[(BlobDigest, &[u8])], + ) -> SynapseResult { + let trailer = if self.revision_seal.is_some() && record.seal().is_none() { + 1 + 32 + 64 + } else { + 0 + }; + let meta = crate::codec::encode_record_with(record, self.codec)?.len() + trailer; + Ok(if self.v2_mode { + inline + .iter() + .fold(8 + meta, |n, (_, b)| n.saturating_add(36 + b.len())) + } else { + meta + }) + } + + /// The Git bridge map of `repo_id`, if anything was imported into it. + pub fn git_bridge_repo(&self, repo_id: &RepoId) -> Option<&GitBridgeRepo> { + self.git_bridge.repo(repo_id) + } + + /// The canonical node bytes of an imported tree (`TreeDigest` to C1 version 1 encoding). + pub fn git_bridge_tree_node(&self, digest: TreeDigest) -> Option<&[u8]> { + self.git_bridge.node(digest) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::codec::{decode_record, encode_record, encode_record_with, RecordCodec}; + use synapse_core::{FileMode, TreeNodeEncoder}; + + const CAP: u64 = 1 << 20; + + fn records() -> Vec { + let repo = RepoId::new("aien-dev/h1-codec").unwrap(); + let body = b"hello\n"; + let blob = BlobDigest::compute(body); + let mut enc = TreeNodeEncoder::new(1); + enc.file("a.txt", FileMode::Regular644, &blob, body.len() as u64) + .unwrap(); + let (tree, node) = enc.finish().unwrap(); + let commit = GitImportedCommit { + git_oid: GitOid::compute_git_object("commit", b"c"), + tree, + parents: vec![ChangesetId(Sha256::digest(b"parent").into())], + headers: b"author a 0 +0000\ncommitter a 0 +0000\n\nm\n".to_vec(), + }; + let entries = vec![ + GitBridgeEntry::Objects { + blobs: vec![GitBlobMapping { + git_oid: GitOid::compute_blob(body), + blob, + size_bytes: body.len() as u64, + }], + trees: vec![GitTreeMapping { + git_oid: GitOid::compute_git_object("tree", b"t"), + node, + }], + }, + GitBridgeEntry::Commit(commit.clone()), + GitBridgeEntry::Tag(GitImportedTag { + git_oid: GitOid::compute_git_object("tag", b"t"), + target: GitBridgeTarget::Commit(commit.changeset_id()), + rest: b"type commit\ntag v1\n\nm\n".to_vec(), + }), + GitBridgeEntry::Refs(GitImportedRefs { + refs: vec![ + ("refs/heads/main".into(), GitBridgeTarget::Tree(tree)), + ("refs/tags/b".into(), GitBridgeTarget::Blob(blob)), + ( + "refs/tags/t".into(), + GitBridgeTarget::Tag(GitOid::compute_git_object("tag", b"t")), + ), + ], + head: Some("refs/heads/main".into()), + }), + GitBridgeEntry::Refs(GitImportedRefs { + refs: Vec::new(), + head: None, + }), + ]; + entries + .into_iter() + .map(|entry| WalRecord::GitBridgeImported { + repo_id: repo.clone(), + entry, + seal: None, + }) + .collect() + } + + #[test] + fn kind_10_round_trips_and_every_truncation_is_refused() { + for record in records() { + let bytes = encode_record(&record).unwrap(); + assert_eq!(bytes[1], 10, "record kind 10"); + assert_eq!(decode_record(&bytes, CAP).unwrap(), record); + for cut in 0..bytes.len() { + assert!( + decode_record(&bytes[..cut], CAP).is_err(), + "a record cut at {cut} of {} bytes decoded", + bytes.len() + ); + } + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(decode_record(&trailing, CAP).is_err(), "trailing byte"); + assert!( + encode_record_with(&record, RecordCodec::JsonV1).is_err(), + "kind 10 has no JSON form" + ); + } + } + + #[test] + fn kind_10_is_a_sealed_revision_record_and_its_digest_excludes_the_seal() { + for mut record in records() { + assert!( + record.is_revision_record(), + "kind 10 is sealed (issue #101)" + ); + let unsealed = encode_record(&record).unwrap(); + let digest = crate::revision::record_digest(&record); + record.set_seal(crate::RevisionSeal { + server_key: [7; 32], + signature: [9; 64], + }); + let sealed = encode_record(&record).unwrap(); + assert_eq!(sealed.len(), unsealed.len() + 1 + 32 + 64, "seal trailer"); + assert_eq!(&sealed[..unsealed.len()], &unsealed[..]); + assert_eq!(decode_record(&sealed, CAP).unwrap(), record); + assert_eq!( + crate::revision::record_digest(&record), + digest, + "the seal covers the digest, so the digest leaves the seal out" + ); + } + } + + #[test] + fn imported_revision_id_covers_tree_parents_and_headers() { + let base = match &records()[1] { + WalRecord::GitBridgeImported { + entry: GitBridgeEntry::Commit(c), + .. + } => c.clone(), + _ => unreachable!(), + }; + let id = base.changeset_id(); + let mut other_oid = base.clone(); + other_oid.git_oid = GitOid::compute_git_object("commit", b"other"); + assert_eq!(other_oid.changeset_id(), id, "the Git ID is not hashed in"); + let mut changed = base.clone(); + changed.headers.push(b'x'); + assert_ne!(changed.changeset_id(), id); + let mut changed = base.clone(); + changed.parents.push(changed.parents[0]); + assert_ne!(changed.changeset_id(), id); + let mut changed = base; + changed.tree = TreeDigest::empty(); + assert_ne!(changed.changeset_id(), id); + } + + /// Issue #101 for kind 10: on a sealed store, a Git bridge record without a valid seal + /// maps nothing. Strict replay refuses the store; torn-tail replay opens it with the + /// record reported unsealed and the repository absent from the bridge map, and the + /// record is left out by compaction. + #[test] + fn kind_10_without_a_seal_maps_nothing_on_sealed_replay() { + use crate::{FsyncPolicy, RecoveryPolicy, RevisionSealKey, StoreLimits, SynapseStore}; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "synapse-h1-unsealed-{}-{nanos}", + std::process::id() + )); + std::fs::create_dir_all(&dir).unwrap(); + let limits = || StoreLimits { + wal_prealloc_bytes: 64 * 1024, + ..Default::default() + }; + let key = || RevisionSealKey::new(ed25519_dalek::SigningKey::from_bytes(&[3; 32])); + let repo = RepoId::new("aien-dev/h1-unsealed").unwrap(); + let body = b"unsealed\n"; + let blob = BlobDigest::compute(body); + let git_oid = GitOid::compute_blob(body); + let entry = GitBridgeEntry::Objects { + blobs: vec![GitBlobMapping { + git_oid, + blob, + size_bytes: body.len() as u64, + }], + trees: Vec::new(), + }; + let mut w = SynapseStore::open( + &dir, + FsyncPolicy::EveryCommit, + RecoveryPolicy::StrictFailClosed, + limits(), + ) + .unwrap(); + assert!(w + .record_git_import(&repo, entry, &[(blob, &body[..])]) + .unwrap() + .is_some()); + assert_eq!( + w.git_bridge_repo(&repo).and_then(|r| r.blob(git_oid)), + Some((blob, body.len() as u64)), + "an unsealed store maps it" + ); + drop(w); + + let sealed = |policy| { + SynapseStore::open_sealed(&dir, FsyncPolicy::EveryCommit, policy, limits(), key()) + }; + assert_eq!( + sealed(RecoveryPolicy::StrictFailClosed).err().unwrap().code, + RefusalCode::RevisionRecordUnsealed + ); + let mut s = sealed(RecoveryPolicy::TruncateTornTail).unwrap(); + assert_eq!(s.seal_replay_report().unsealed, 1); + assert!( + s.git_bridge_repo(&repo).is_none(), + "an unsealed kind 10 record mapped a Git ID on sealed replay" + ); + s.checkpoint_and_compact(1_000).unwrap(); + drop(s); + let s = sealed(RecoveryPolicy::StrictFailClosed).expect("strict accepts the compacted store"); + assert!(s.git_bridge_repo(&repo).is_none()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/synapse-store/src/j1_corpus.rs b/crates/synapse-store/src/j1_corpus.rs index 2eeff67..f19cc70 100644 --- a/crates/synapse-store/src/j1_corpus.rs +++ b/crates/synapse-store/src/j1_corpus.rs @@ -20,7 +20,7 @@ use synapse_core::{ChangeId, FileMode, PathMutation, RevisionContent, TreeDigest /// The sealed corpus, relative to the repository root. const CORPUS_PATH: &str = "evidence/j1-codec-corpus/corpus.bin"; /// SHA-256 of the sealed corpus, so the seal is checked by a test, not by hand. -const CORPUS_SHA256: &str = "469a25239b6d683d137497b8cab1bebb186e11b180fb5061723af2fe8f6bb8b7"; +const CORPUS_SHA256: &str = "2597e5ca9b9dfc80be67cdc7436143df462a42bc54e1504d38656730d7eb5ca6"; const MAGIC: [u8; 8] = *b"SYNJ1COD"; const VERSION: u16 = 1; /// Record cap of the A1 malformed corpus (`a1_malformed_corpus.rs`). @@ -288,10 +288,12 @@ impl Lf { self.chain(chain); self.u64(*recorded_at_ms); } - // Kinds 9 and 11 landed on main after this corpus was sealed; the corpus covers + // Kinds 9 to 11 landed on main after this corpus was sealed; the corpus covers // kinds 1 to 8 only (see the coverage test), so no corpus record reaches here. - WalRecord::RevisionStateMoved { .. } | WalRecord::DerivedRevisionAdmitted { .. } => { - unreachable!("J1 corpus does not yet cover record kinds 9 and 11") + WalRecord::RevisionStateMoved { .. } + | WalRecord::GitBridgeImported { .. } + | WalRecord::DerivedRevisionAdmitted { .. } => { + unreachable!("J1 corpus does not yet cover record kinds 9 to 11") } } } @@ -515,8 +517,9 @@ fn bases() -> Vec { } let fixture = checkpoint_base("fixture checkpoint", &fixture_state()); let revision = checkpoint_base("revision checkpoint", &revision_state()); - // Hand-made inputs no mutation reaches: record kinds 9 to 11 (reserved by open PRs, refused - // on `main`), a revision section that is empty, and one that holds a kind 6 record. + // Hand-made inputs no mutation reaches: record kinds 9 to 11 with a cut body (refused; all + // three kinds decode on `main` since B4, H1 and B5), a revision section that is empty, and + // one that holds a kind 6 record. for kind in 9u8..=11 { out.push(refused_base( format!("reserved record kind {kind}"), diff --git a/crates/synapse-store/src/lib.rs b/crates/synapse-store/src/lib.rs index f48956f..465e969 100644 --- a/crates/synapse-store/src/lib.rs +++ b/crates/synapse-store/src/lib.rs @@ -36,6 +36,7 @@ mod blockdev_a2_suite; mod blockdev_tests; pub mod chunk; pub mod codec; +pub mod git_bridge; #[cfg(test)] mod g2_tests; #[cfg(test)] @@ -368,6 +369,14 @@ pub enum WalRecord { admitted_at_ms: u64, seal: Option, }, + /// One Git bridge entry of an import into `repo_id` (ADR 0003 section 5.10, step H1 cut 2; + /// `git_bridge`). Binary codec only. Sealed like the other revision records (issue #101). + #[serde(skip)] + GitBridgeImported { + repo_id: RepoId, + entry: git_bridge::GitBridgeEntry, + seal: Option, + }, } impl WalRecord { @@ -382,17 +391,19 @@ impl WalRecord { Self::RevisionAdmitted { .. } => 7, Self::ObsolescenceRecorded { .. } => 8, Self::RevisionStateMoved { .. } => 9, + Self::GitBridgeImported { .. } => 10, Self::DerivedRevisionAdmitted { .. } => 11, } } - /// True for the revision records a seal covers (kinds 7, 8, 9, and 11). + /// True for the revision records a seal covers (kinds 7 to 11). pub fn is_revision_record(&self) -> bool { matches!( self, Self::RevisionAdmitted { .. } | Self::ObsolescenceRecorded { .. } | Self::RevisionStateMoved { .. } + | Self::GitBridgeImported { .. } | Self::DerivedRevisionAdmitted { .. } ) } @@ -403,6 +414,7 @@ impl WalRecord { Self::RevisionAdmitted { seal, .. } | Self::ObsolescenceRecorded { seal, .. } | Self::RevisionStateMoved { seal, .. } + | Self::GitBridgeImported { seal, .. } | Self::DerivedRevisionAdmitted { seal, .. } => seal.as_ref(), _ => None, } @@ -412,15 +424,18 @@ impl WalRecord { if let Self::RevisionAdmitted { seal, .. } | Self::ObsolescenceRecorded { seal, .. } | Self::RevisionStateMoved { seal, .. } + | Self::GitBridgeImported { seal, .. } | Self::DerivedRevisionAdmitted { seal, .. } = self { *seal = Some(value); } } - /// Blob digests the upserts of this record reference (ingest and revision records). + /// Blob digests the upserts of this record reference (ingest and revision records), and the + /// blobs a Git bridge objects entry maps. pub fn upsert_blobs(&self) -> Vec { let mutations = match self { + Self::GitBridgeImported { entry, .. } => return entry.blobs(), Self::ChangesetIngest { envelope, .. } => &envelope.mutations, Self::RevisionAdmitted { envelope, .. } => &envelope.content.mutations, Self::DerivedRevisionAdmitted { derived, .. } => &derived.content.mutations, @@ -555,6 +570,8 @@ pub struct SynapseStore { /// `checkpoint.v1` is not yet the tombstone although `checkpoint.v2` exists. /// Envelope v2 revisions, changes, witnesses, and markers (ADR 0003 B1). revisions: revision::RevisionLedger, + /// The Git bridge map (step H1 cut 2), derived from the kind 10 records in `revisions`. + git_bridge: git_bridge::GitBridgeLedger, /// The server key revision records are sealed with and checked against (issue #101). revision_seal: Option, tombstone_pending: bool, @@ -674,8 +691,8 @@ impl SynapseStore { } /// [`Self::open`] with the server's machine key as the revision seal key (issue #101). - /// Every revision record written (kinds 7, 8, 9, and 11) is sealed by `seal`, chained to - /// the previous one; on replay a revision record whose seal does not verify against that + /// Every revision record written (kinds 7, 8, 9, 10, and 11) is sealed by `seal`, chained + /// to the previous one; on replay a revision record whose seal does not verify against that /// chain changes nothing. `StrictFailClosed` refuses such a store with /// `RevisionRecordUnsealed` (including one with revision records written before it was /// sealed); `TruncateTornTail` opens it, reports them in [`Self::seal_replay_report`], @@ -834,6 +851,7 @@ impl SynapseStore { revisions: revision::RevisionLedger::new( revision_seal.as_ref().map(RevisionSealKey::public_key), ), + git_bridge: git_bridge::GitBridgeLedger::default(), revision_seal, }; @@ -2057,6 +2075,7 @@ impl SynapseStore { } } } + retained_blobs.extend(self.git_bridge.blob_digests()); for stored in self.changesets.values() { for m in &stored.envelope.mutations { if let MutationOp::Upsert { blob_digest, .. } = &m.op { @@ -2620,11 +2639,19 @@ impl SynapseStore { } record @ (WalRecord::ObsolescenceRecorded { .. } | WalRecord::RevisionStateMoved { .. }) => self.revisions.apply(seq, record), - record @ WalRecord::DerivedRevisionAdmitted { .. } + record @ (WalRecord::DerivedRevisionAdmitted { .. } + | WalRecord::GitBridgeImported { .. }) if !self.revisions.accepts(seq, &record) => { self.revisions.apply(seq, record); } + record @ WalRecord::GitBridgeImported { .. } => { + for digest in record.upsert_blobs() { + self.blob_first_visible_seq.entry(digest).or_insert(seq); + } + self.git_bridge.apply(&record); + self.revisions.apply(seq, record); + } record @ WalRecord::DerivedRevisionAdmitted { .. } => { for digest in record.upsert_blobs() { self.blob_first_visible_seq.entry(digest).or_insert(seq); @@ -2750,6 +2777,7 @@ impl SynapseStore { s.revision_log, self.revision_seal.as_ref().map(RevisionSealKey::public_key), ); + self.git_bridge = git_bridge::GitBridgeLedger::from_log(self.revisions.applied_log()); } /// Admit a replayed (not checkpoint-covered) frame written with `codec`: the codec diff --git a/crates/synapse-store/src/revision.rs b/crates/synapse-store/src/revision.rs index f8af22c..728ea4f 100644 --- a/crates/synapse-store/src/revision.rs +++ b/crates/synapse-store/src/revision.rs @@ -184,7 +184,7 @@ pub(crate) struct CompactedRevisionLog { } /// Digest of one revision record over its canonical core encodings. -fn record_digest(record: &WalRecord) -> [u8; 32] { +pub(crate) fn record_digest(record: &WalRecord) -> [u8; 32] { let mut h = Sha256::new(); h.update([record.kind_byte()]); let mut chain = Vec::new(); @@ -239,6 +239,9 @@ fn record_digest(record: &WalRecord) -> [u8; 32] { h.update(m.mover_key); h.update(m.signature); } + WalRecord::GitBridgeImported { repo_id, entry, .. } => { + crate::git_bridge::record_digest_input(repo_id, entry, &mut h); + } WalRecord::DerivedRevisionAdmitted { derived, chain: c, @@ -283,6 +286,14 @@ impl RevisionLedger { &self.log } + /// The log without the records whose seal did not verify (issue #101): the records that + /// took effect, in sequence order. + pub(crate) fn applied_log(&self) -> impl Iterator { + self.log + .iter() + .filter(|(seq, _)| !self.unsealed.contains(seq)) + } + /// Kind 9 records that changed nothing, by reason (issue #89). pub(crate) fn state_move_report(&self) -> StateMoveReplayReport { self.state_moves @@ -541,6 +552,9 @@ impl RevisionLedger { } } } + // Git bridge records are kept in the log (checkpoint and state root); their map + // lives in `git_bridge::GitBridgeLedger`. + WalRecord::GitBridgeImported { .. } => {} _ => return, } self.log.push((seq, record)); @@ -1042,6 +1056,13 @@ impl crate::SynapseStore { // Another owner's `Private` revision is not a parent this chain can name (B4). let parent_repo = if self.revisions.get_visible(repo, *parent, viewer).is_some() { repo + } else if self + .git_bridge + .repo(repo) + .is_some_and(|r| r.commit_by_changeset(*parent).is_some()) + { + // An imported Git commit (step H1) of this repository. + repo } else if let Some(cs) = self.changesets.get(parent) { &cs.envelope.repo_id } else { @@ -1274,7 +1295,7 @@ impl crate::SynapseStore { .ok_or_else(|| not_found(format!("change {change_id} not found"))) } - fn check_binary_codec(&self) -> SynapseResult<()> { + pub(crate) fn check_binary_codec(&self) -> SynapseResult<()> { if self.codec == crate::RecordCodec::JsonV1 { return Err(SynapseError::new( RefusalCode::EnvelopeVersionUnsupported, @@ -1291,7 +1312,7 @@ impl crate::SynapseStore { /// before anything is written, and a frame that lands at another sequence anyway is not /// applied and poisons the store (`WalSequenceGap` in both cases), so a record is never /// admitted at a sequence it was not checked at. Real checks in every build profile. - fn append_revision_record( + pub(crate) fn append_revision_record( &mut self, mut record: WalRecord, blobs: &[(BlobDigest, &[u8])], diff --git a/crates/synapse-store/src/seal.rs b/crates/synapse-store/src/seal.rs index 7eaeb69..5ef0fa3 100644 --- a/crates/synapse-store/src/seal.rs +++ b/crates/synapse-store/src/seal.rs @@ -5,7 +5,7 @@ //! Kinds 7 and 8 carry a `chain` field that no signature in the record covers, so such a //! rewrite could swap an admission's chain for links the forger minted and then sign kind 9 //! moves under them. A store opened with a [`RevisionSealKey`] (the server's machine key) -//! closes that: every revision record it writes (kinds 7, 8, 9, and 11) carries a +//! closes that: every revision record it writes (kinds 7, 8, 9, 10, and 11) carries a //! [`RevisionSeal`], an Ed25519 signature by that key over a running head that chains the //! record's `WalSeq`, its full record digest (every field, chain included), and the head of //! the previous sealed record. Replay applies a revision record only when its seal verifies diff --git a/crates/synapse-store/tests/a1_malformed_corpus.rs b/crates/synapse-store/tests/a1_malformed_corpus.rs index c90fbcd..df7536d 100644 --- a/crates/synapse-store/tests/a1_malformed_corpus.rs +++ b/crates/synapse-store/tests/a1_malformed_corpus.rs @@ -162,6 +162,8 @@ fn length_fields(record: &WalRecord, bin: &[u8]) -> Vec { | WalRecord::DerivedRevisionAdmitted { .. } => vec![2], // repo_id length first (ADR 0003 B4). WalRecord::RevisionStateMoved { .. } => vec![2], + // repo_id length first (ADR 0003 H1, kind 10; not in the v1 fixture). + WalRecord::GitBridgeImported { .. } => vec![2], } } diff --git a/docs/H1_GIT_BRIDGE.md b/docs/H1_GIT_BRIDGE.md index 359d773..1f787f2 100644 --- a/docs/H1_GIT_BRIDGE.md +++ b/docs/H1_GIT_BRIDGE.md @@ -1,10 +1,10 @@ -# H1: Git Bridge, Cut 1 of 2: Native Git Objects, Packfiles, Full-History Read and Export +# H1: Git Bridge: Native Git Objects, the Persistent Map, Import and Export ADR 0003 section 9 step 28, H1 (issue aien-spark/synapse#48). Section 5.10: "Import: full history into native objects, with a persistent `GitOid` to (`ChangesetId`, `TreeDigest`) map; merge parents, tags, modes, and signatures preserved (signatures as opaque data). Export: ... written as packfiles, not loose objects". Section 4 adapts Git packfiles; section 6 keeps deltas "only on the bridge wire". -H1 is delivered in two ordered cuts. This document covers **cut 1**, the Git side of the bridge: reading a real repository's whole object graph natively and writing an object set back as a pack that Git accepts byte for byte. **Cut 2** (not in this change) adds the persistent map and the native side: imported blobs into the store, Git trees to `TreeDigest`, commits to revisions with their original headers kept as opaque bridge data, and export of imported and native history from that stored state. +H1 is delivered in two ordered cuts. **Cut 1** (sections 1 and 1a) is the Git side of the bridge: reading a real repository's whole object graph natively and writing an object set back as a pack that Git accepts byte for byte. **Cut 2** (section 1b) is the native side: the persistent `GitOid` to (`ChangesetId`, `TreeDigest`) map in the store (WAL record kind 10), imported blobs as store blobs, Git trees as canonical `TreeDigest` nodes, commits as imported revisions keeping their original headers as opaque bytes, export of that stored state with identical Git object IDs, and deterministic export of native (B1) revisions. -Code: `crates/synapse-git/src/{object,inflate,pack,repo}.rs`. Tests: `crates/synapse-git/tests/h1_git_bridge.rs` and unit tests in each module. Evidence: `evidence/h1-git-bridge/`. +Code: `crates/synapse-git/src/{object,inflate,pack,repo,bridge}.rs` and `crates/synapse-store/src/git_bridge.rs`. Tests: `crates/synapse-git/tests/{h1_git_bridge,h1_hostile_repo,h1_read_limits,h1_bridge_import}.rs` and unit tests in each module. Evidence: `evidence/h1-git-bridge/`. ## 1. What cut 1 provides @@ -37,15 +37,86 @@ All five limits are fields of `ReadLimits` (`GitRepository::open_with_limits`); A file swapped between the check and the open is refused too: the open uses `O_NOFOLLOW | O_NONBLOCK` (`OpenOptionsExt::custom_flags`, constants from the workspace `libc` dependency), so a swapped-in link fails with `ELOOP` and a swapped-in pipe opens at once and fails the device, inode and regular-file check (unit test `a_file_swapped_for_a_pipe_or_link_after_the_check_is_refused_without_blocking`). +Re-review of PR #90 at `28d783c` (non-blocking findings, folded into cut 2): (a) a fan-out directory under `objects/` (`objects/ab/`) that is a symbolic link could lead reads outside the repository; it must now be a real directory like `objects/` itself, or the read is refused `PathInvalid` (`h1_hostile_linked_object_fanout_directory_is_refused`). Directories under `refs/` were already refused unless real (the walk's `DirEntry::file_type` does not follow links, and anything that is neither a directory nor a regular file is `PathInvalid`). (b) The standalone pack indexer (`PackReader::index`, native `index-pack`) now goes through `ReadLimits` too (`PackReader::index_with_limits`; `index` uses the defaults): the declared size of every entry is summed against `max_total_object_bytes` before anything is resolved, and reference deltas whose base comes later in the pack wait on that base instead of being retried in rounds (unit test `indexer_resolves_a_reference_delta_stored_before_its_base_within_the_budget`: the exact budget passes, one byte less is refused, the per-object cap is refused, a thin pack is refused). (c) Peak RSS against the budget is stated in section 4. + +## 1b. Cut 2: the persistent map, native import, export from the store + +### WAL record kind 10 + +`WalRecord::GitBridgeImported { repo_id, entry, seal }`, WAL record **kind 10**, binary codec only (no v1 JSON form; a store still writing v1 refuses it as kinds 7 and 8 are refused). Kinds 7, 8, 9 and 11 are the B1, B4 and B5 revision records, so the bridge takes 10. Body: `repo_id | entry tag (u8) | entry body | optional seal trailer` (layout in `codec.rs` beside `put_git_bridge_entry`; the trailer is issue #101's `1 | server key [32] | signature [64]`, as on kinds 7, 8, 9 and 11): + +| Entry | Tag | Body | +| :--- | ---: | :--- | +| `Objects` | 1 | `u32 n \| (git oid [20] \| blob [32] \| size u64)* \| u32 m \| (git oid [20] \| u32 len \| C1 tree node)*` | +| `Commit` | 2 | `git oid [20] \| tree [32] \| u32 n \| parent ChangesetId [32]* \| u32 len \| headers` | +| `Tag` | 3 | `git oid [20] \| target \| u32 len \| rest` | +| `Refs` | 4 | `u32 n \| (name \| target)* \| option HEAD` | + +A target is `1 | ChangesetId`, `2 | TreeDigest`, `3 | BlobDigest` or `4 | tag git oid`. Blob bytes travel inline in the WAL frame as for every v2 record (only blobs not already committed). Replay, checkpoints and compaction treat kind 10 like the other revision records: the records stay in the revision log, a checkpoint load rebuilds the map from the records that took effect (`GitBridgeLedger::from_log(RevisionLedger::applied_log())`), the state root covers them, and compaction keeps every blob the map names. + +In a store opened with a seal key (`SynapseStore::open_sealed`), kind 10 is sealed like the other revision records (issue #101): the write seals it for the exact sequence it lands at, the record digest the seal signs is taken over `repo_id | entry` without the trailer, and replay applies a kind 10 record (its blobs and its map entries) only when its seal verifies against the chain so far. An unsealed or forged record stays in the log and the state root, changes nothing, is counted in the seal report, is refused by strict recovery, and is dropped by compaction. + +### The map + +`SynapseStore::git_bridge_repo(repo)` gives, per repository: Git blob to (`BlobDigest`, size); Git tree to `TreeDigest` (the canonical C1 node is stored, so the tree is rebuilt at export, never kept as Git bytes); Git commit to (`ChangesetId`, `TreeDigest`) and back; Git tag to its target and remaining bytes; the references and `HEAD`. An imported commit is a revision whose ID is `SHA-256("SYNAPSE_GIT_IMPORTED_REVISION_V1\0" || u64 len || version 1 | tree | parents | headers)`: the tree, the parents in written order and every byte after the `tree` and `parent` lines (author, committer, `gpgsig`, `mergetag`, `encoding`, message) byte for byte. The Git ID is not hashed in, so it is checked, not trusted: every export recomputes it. A B1 revision may name an imported commit of its repository as a parent. + +`SynapseStore::record_git_imports(repo, records)` is the only writer (`record_git_import` is a batch of one). It checks the whole batch before the first write: each record is run, in order, against a copy of the map in which the batch's earlier records are already applied, and only when every record passes are they written, each checked again against the live map. So any refusal below leaves the store unchanged, however late in the batch the refused record sits (review of #96, B1). The checks, per record: + +* a blob over `max_blob_bytes`, the record's blobs over `max_total_changeset_bytes`, or the record's exact WAL frame (metadata plus inline blobs, encoded as the write will encode it) over `max_wal_record_bytes`: `QuotaPayloadTooLarge`; +* a blob whose bytes or declared size do not match (`BlobDigestMismatch`), or neither provided nor committed (`BlobNotFound`); +* a Git ID mapped again to different content, a Git ID twice in one entry, or a second Git ID for the content of a commit already imported: `GitObjectCorrupt`; +* every Git ID the record newly maps is re-hashed with the workspace SHA-1 from what it maps to, and a mismatch is `GitObjectCorrupt` (review of #96, B2): a blob from its bytes; a tree from its C1 node rebuilt as a Git tree (`synapse_core::git_tree_payload_of_node`, the same function export uses) with the Git IDs of its entries; a commit from its `tree` and `parent` lines plus its stored headers; a tag from its `object` line plus its stored bytes. Entries already mapped were checked when written; +* a non-canonical tree node (`TreeNodeMalformed`); a tree file entry naming a blob that no Git blob of the same repository maps (`BlobNotFound`) or with another size (`BlobDigestMismatch`); a subtree not imported into the same repository (`TreeNodeNotFound`); +* a commit on a tree not imported into the repository (`TreeNodeNotFound`) or on an unknown parent (`ParentChangesetNotFound`); +* a reference name that is not a safe `refs/...` path, out of order, or both a file and a directory (`PathInvalid`); a tag or reference target not imported into the repository (`GitObjectCorrupt`). + +An entry already fully recorded writes nothing, so importing the same history twice writes no record. + +### Import + +`import_repository(store, repo, git_dir, limits)` reads the full history with the cut 1 reader (bounded by `ReadLimits`), converts all of it in memory, and hands every record to `record_git_imports` as one batch: `Objects` records (at most 4,096 mappings, `max_total_changeset_bytes` of inline bytes, and an upper bound of the WAL frame within `max_wal_record_bytes` each), one `Commit` record per commit (parents first), one `Tag` record per tag (inner tags first), one `Refs` record. A repository with anything that has no exact native form is refused `GitObjectNotRepresentable` before the first record, leaving the store unchanged: + +* tree entry modes other than `100644`, `100755`, `120000` and `40000` (gitlinks `160000`, legacy `100664`, zero-padded `040000`); +* entry names that are not UTF-8 or that C1 refuses (`.git`, `.`, `..`, a `/` or `\`, empty, over 2,048 bytes); +* an empty subtree, or a tree whose canonical node does not rebuild the same Git tree (order, duplicates); +* a commit whose bytes do not start with exactly its `tree` and `parent` lines, or a tag without its leading `object` line. + +A record the store's limits refuse (a blob over `max_blob_bytes` or `max_total_changeset_bytes`; a commit, tag or single wide tree whose frame is over `max_wal_record_bytes`) is `QuotaPayloadTooLarge` before the first record, and the reference names are checked by the same rule the store applies, also before the first record. + +### Export + +`export_store(store, repo, native_refs, dest)` writes a new bare repository from the store alone. Every imported blob, tree, commit and tag is rebuilt from the map and the stored blobs and must hash to its imported Git ID (`GitObjectCorrupt` otherwise); the imported references and `HEAD` follow. Each entry of `native_refs` adds a reference to a native revision, written with its native ancestors by this rule: + +* tree: the revision's `base_tree` (the empty tree, an imported tree, or the result tree of a native revision exported before it) with its mutations applied as the tree store applies them (an upsert replaces a file at an ancestor path and everything below its own path; a delete removes a file); +* parents in the revision's order (imported parents by their Git IDs); +* `author 0 +0000`, `committer Synapse Engine 0 +0000`; +* message: the intent summary, a blank line, `Synapse-Changeset-Id: ` and `Synapse-Change-Id: `. + +Nothing depends on the clock or on order of insertion, so the same store state gives the same pack bytes. A native reference that names an imported reference is `PathInvalid`. + ## 2. Acceptance status (issue #48) | Acceptance (issue #48) | Status | Evidence | | :--- | :--- | :--- | -| Imported real repository history with no erased content exports with identical Git object IDs, using the stored original headers | Git side done: real full history read natively and exported with identical IDs. The native import and the stored-header path are cut 2 | `h1_full_history_exports_as_a_pack_with_identical_object_ids_and_fsck_strict_clean`; real repositories below | -| Native revisions export deterministically (two exports give identical IDs) | Pack writer deterministic (two exports, either input order, give identical pack and index bytes). Native revisions are cut 2 | `h1_export_is_deterministic_and_never_overwrites_a_repository` | -| `git fsck --full --strict` clean | Done for every exported repository, fixture and real | same tests | +| Imported real repository history with no erased content exports with identical Git object IDs, using the stored original headers | Done: full history imported into the store (blobs, C1 trees, imported revisions with their original headers, tags, references), then exported from the store alone with identical IDs, for the fixture and two real repositories; the map survives reopen, checkpoint and compaction with the same export | `h1_import_then_export_from_the_store_gives_identical_ids_refs_and_fsck_strict_clean`, `h1_bridge_map_survives_reopen_and_checkpoint_with_the_same_export`, `h1_real_repository_round_trips_through_the_store_with_identical_ids` (ignored by default; real repositories below) | +| Native revisions export deterministically (two exports give identical IDs) | Done: a native revision on imported history (a file edited, a file deleted, a file turned into a directory) exports to the same pack twice and after a reopen; a native child of it exports on its parent's result tree | `h1_native_revisions_export_deterministically_on_imported_history`, `h1_a_native_child_of_a_native_revision_exports_on_its_parents_result_tree` | +| `git fsck --full --strict` clean | Done for every exported repository, fixture and real, cut 1 and cut 2 | same tests | + +Cut 2 hostile input at the import boundary (`h1_import_refuses_*`, including a record over the store's limits, `h1_store_refuses_crafted_bridge_entries`, `h1_store_refuses_git_ids_that_do_not_match_their_content`): each case is refused typed with the store's committed sequence unchanged and no map created; reference names that would write outside the export directory never reach a write. + +Real repositories through the store (`evidence/h1-git-bridge/red-green/cut2-green.txt`): the same mirror clones, imported into a fresh store (`max_blob_bytes` raised to 1 GiB), the store reopened (WAL replay), exported from the store, then the cut 1 Git checks. Release test binary, `/usr/bin/time -v`: + +| Repository | Objects (blobs, trees, commits, tags) | References | Records | Import | Export | Peak RSS | Result | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | :--- | +| `aien-dev/synapse` (main `3d78efe`) | 1,915 (876, 859, 180, 0) | 75 | 183 | 7.5 s | 1.0 s | 197,116 KiB | identical IDs and references, fsck strict clean | +| `aien-dev/omega` (main `f4a9849`) | 29,636 (16,867, 10,345, 2,413, 11) | 633 | 2,433 | 123.3 s | 5.8 s | 976,824 KiB | identical IDs and references, fsck strict clean | + +Reopening the store (WAL replay of the records) took 0.1 s and 0.3 s. The cut 2 mirrors are newer clones (same `main`, more pull request references), so the counts differ from the cut 1 table. + +After the review of #96 (whole-import check, Git ID re-hash; `review96-green.txt`), the same two mirrors again: identical IDs and references, fsck strict clean, the same object and record counts. synapse: import 8.1 s, reopen 0.1 s, export 0.9 s, peak RSS 197,056 KiB. omega: import 133.7 s, reopen 4.1 s, export 15.1 s, peak RSS 977,812 KiB. The host was shared during that run (load average about 14.5 on 20 cores), so these times are not a clean comparison with the table above; peak RSS is 0.101% above it (calc:f87ba615b0d5). + +Cut 1 runs (`evidence/h1-git-bridge/red-green/green.txt`), each a `git clone --mirror` (full history, every reference including pull request refs): native read, native export, then Git checks that every reference resolves to the same object, that the exported database holds exactly the source's reachable objects (`rev-list --objects --all`), and `fsck --full --strict`: -Real repositories (`evidence/h1-git-bridge/red-green/green.txt`), each a `git clone --mirror` (full history, every reference including pull request refs): native read, native export, then Git checks that every reference resolves to the same object, that the exported database holds exactly the source's reachable objects (`rev-list --objects --all`), and `fsck --full --strict`: | Repository | Objects | Commits | References | Result | | :--- | ---: | ---: | ---: | :--- | @@ -69,11 +140,44 @@ The fixture (built by Git in the test) covers nested trees, executable files, a `evidence/h1-git-bridge/red-green/hostile-red.txt` (review findings B1 and B2): `tests/h1_hostile_repo.rs` uses only `GitRepository::open` and `full_history`, so the same file runs unchanged against the reviewed head `b4fe463`. There all 8 cases fail: the pipe cases (a reference, `packed-refs`, a loose object) block until the test's 10 s deadline releases them, the `/dev/zero` link fails with `IoFailure: out of memory` under the run's `ulimit -v 4000000` guard, the link to a file outside the repository and the 1 MiB reference are read as references, the 82 KB delta bomb resolves 5 objects of 256 MiB, and the detached `HEAD` commit is missing. Three further stubs on the fixed code: (A) the total budget check removed: the budget test fails; (B) links followed (`metadata` instead of `symlink_metadata`, and in the `refs/` walk): the outside-link test fails; (C) files opened without `O_NOFOLLOW | O_NONBLOCK`: the swap test blocks on the pipe until its 10 s deadline. `hostile-green.txt` holds the passing runs. +Re-review of #90 (`rereview-red.txt`): stub N1 (fan-out directory check removed) makes `h1_hostile_linked_object_fanout_directory_is_refused` fail; stub N3 (indexer budget check removed) makes the indexer unit test fail ("one byte short: indexed 2 entries instead of refusing"). + +Cut 2 (`cut2-red.txt`): seven stubs, each removing one behaviour, each applied alone to the finished code; the named test fails by assertion or by the test's own typed check, never by a compile error: + +1. S1 the canonical-form check skipped (legacy and zero-padded modes accepted, tree rebuild not compared): the no-native-form test fails, "legacy mode 100664: imported instead of refusing". +2. S2 commit headers dropped on import: the round trip fails, export refused `GitObjectCorrupt` (the stored commit rebuilds under another ID). +3. S3 the map not rebuilt on a checkpoint load: the persistence test fails, "a checkpoint load rebuilds the same map". +4. S4 native commit time taken from the clock: the determinism test fails, "two exports give the same pack". +5. S5 reference names left to the store's check (after the objects are written): the reference test fails, "dot-dot path: nothing written" (`WalSeq(2)` against `WalSeq(0)`). +6. S6 compaction drops the blobs the map holds: the persistence test fails, export refused `BlobNotFound`. +7. S7 native result trees not remembered: the native child test fails, export refused `TreeNodeNotFound`. + +`cut2-green.txt` holds the passing workspace run and the two real repository runs through the store. + +Review of #96 (`review96-red.txt`). Baseline: the two new tests, run unchanged against the reviewed head `b69db2d`, both fail: "one Git ID mapped to two blobs in one entry: accepted", and "commit record over max_wal_record_bytes: nothing written" (`WalSeq(1)` against `WalSeq(0)`: the blobs were written before the commit was refused). Eight stubs on the fixed code, each alone, each failing by assertion: + +1. R1 the dry run skips the frame size: the limits test fails, nothing-written assertion (`WalSeq(1)` against `WalSeq(0)`). +2. R2 the importer writes record by record instead of one checked batch: the same failure. +3. R3 blob Git IDs not re-hashed: "a blob under another blob's Git ID: accepted". +4. R4 tree Git IDs not re-hashed: "a tree under a wrong Git ID: accepted". +5. R5 commit Git IDs not re-hashed: "a commit under a wrong Git ID: accepted". +6. R6 tag Git IDs not re-hashed: "a tag under a wrong Git ID: accepted". +7. R7 a tree file entry may name any visible blob (N3): the tree from another repository is then refused only later, with `GitObjectCorrupt` instead of `BlobNotFound`. +8. R8 export passes any `Tag(oid)` target through: the export unit test fails (the lookup returns a Git ID). + +Two checks have no stub of their own because the re-hash already refuses every case they catch: a Git ID twice in one entry (two different contents cannot both hash to it; the identical duplicate is harmless) and a second Git ID for one commit's content (it cannot hash to both). Both are kept so the refusal names the fault. The same tests also show a correct import into the same store after each refusal (the limits test imports the full repository under the default limits; the Git ID test imports the fixture into the repository that took the refusals), each exported with identical IDs and fsck strict clean. `review96-green.txt` holds the passing runs. + + ## 4. Limits and what remains -* **Cut 2 remains**: the persistent `GitOid` to (`ChangesetId`, `TreeDigest`) map; imported blobs stored natively; Git trees converted to `TreeDigest` and recomputed at export; commits as revisions whose original commit and tag headers are kept as opaque bridge data; export of imported history from stored headers and of native revisions with deterministic generated message and author fields (section 5.10). It needs a new WAL record kind; open PR #88 claims kind 9, so cut 2 will use kind 10. -* Exported packs are not compressed beyond zlib stored blocks and hold no deltas: the omega export is 374,712,257 bytes against 50,117 KiB for Git's own pack (`count-objects -v`). Correct and accepted by Git, but large; a native compressor is a later change. +* Kinds 7, 8, 9 and 11 are the other revision records on main; kind 10 was reserved for the bridge and nothing here depends on the others. +* A detached `HEAD`'s commit is imported (with its history) but no reference names it, and the exported `HEAD` then names `refs/heads/main` (the exporter's default); `git fsck` accepts it, the objects are identical, but the detached state is not reproduced. +* A native revision exports only when its `base_tree` is the empty tree, an imported tree, or the result of a native revision exported in the same call (its parent chain). A base registered only in the API tree store (`register_base_tree`) is `TreeNodeNotFound`. Native references are passed by the caller; the store keeps no native reference state in this cut. Native commits carry no signature. +* An import refused by the checks writes nothing. An I/O failure part way through the writes leaves the records written so far; importing again skips them and writes the rest (each record is checked against the map), so the import resumes, but the store is between states until then. +* In a store opened without a seal key, WAL replay applies kind 10 records without re-running the write checks (`GitBridgeLedger::apply` is total, as kinds 7 and 8 replay), so a well-formed but crafted frame placed in the WAL by hand is trusted by the map. A sealed store refuses that frame unless it carries a valid server seal (issue #101). Export still re-hashes every blob, tree, commit and tag against its Git ID, refuses a `Tag(oid)` target that is not an imported tag, and checks reference names again, so such a frame cannot write outside the export directory or export an object under a wrong ID (review of #96, N1). +* The fan-out and `refs/` directories are checked with `symlink_metadata` and then the file below is opened with `O_NOFOLLOW`, which guards only the last path component: a directory swapped for a link between the check and the open would be followed. Closing that needs a walk relative to opened directory handles (`openat` with `O_DIRECTORY | O_NOFOLLOW` per component); not in this change. +* Exported packs are not compressed beyond zlib stored blocks and hold no deltas: the omega export is 374,712,257 bytes in cut 1 and 374,713,444 bytes in the cut 2 run through the store against 50,117 KiB for Git's own pack (`count-objects -v`). Correct and accepted by Git, but large; a native compressor is a later change. * SHA-1 object format only. A SHA-256 Git repository (`extensions.objectformat = sha256`) is not read. -* The existing per-changeset `materialize_changeset_to_git` still writes loose objects; it is a legacy path and is replaced by the cut 2 export. -* `GitRepository` reads whole pack files into memory and `full_history` holds every object, both bounded by `ReadLimits` (section 1a). A repository larger than the defaults needs the caller to raise them. Peak RSS of the test process for the omega round trip with the defaults: 1,114,576 KiB (`/usr/bin/time -v`). -* No performance figure is claimed: none is part of this cut's acceptance and no run was measured against a section 6 bound. +* The existing per-changeset `materialize_changeset_to_git` still writes loose objects; it is a legacy path, `export_store` is the bridge's export, and removing the legacy path is not part of this change. +* **Memory against the budget (review item c).** `ReadLimits::max_total_object_bytes` (default 1 GiB = 1,048,576 KiB) bounds the resolved object bytes a read holds, not the whole process: pack files are held whole, and the history map, the import's converted records and the export's object set sit beside them. Measured peak RSS (`/usr/bin/time -v`) for omega (374,301,457 bytes of objects): cut 1 read and export 1,114,576 KiB, 1.063 times the budget (calc:f4b7563320d5) and 3.049 times the object bytes (calc:b42240ae452f); cut 2 import, reopen and export through the store 976,824 KiB, 0.9316 times the budget (calc:3499282d5a7c) and 2.672 times the object bytes (calc:15f1957f5764). After the review of #96 the import checks the whole batch on a copy of the map (the copy holds every repository's map and tree nodes while the batch is checked); the omega peak was 977,812 KiB, 0.9325 times the budget (calc:54368b37d67c) and 2.675 times the object bytes (calc:51cc5db6ffa7). The breakdown between those parts was not profiled. Delta streams are not counted either: resolving a deep delta chain holds up to 4,096 inflated delta streams, and the budget counts only declared result sizes, while copy instructions can make a stream several times larger than its result (review of #96, N2; unchanged from cut 1). A repository larger than the defaults needs the caller to raise them. +* No performance figure is claimed against a section 6 bound; the import and export times above are measurements of the test runs, not profiled (the test store uses `FsyncPolicy::EveryCommit`, one WAL record per imported commit). diff --git a/docs/SCHEMA_AND_CONSISTENCY.md b/docs/SCHEMA_AND_CONSISTENCY.md index 2e0480d..b695da5 100644 --- a/docs/SCHEMA_AND_CONSISTENCY.md +++ b/docs/SCHEMA_AND_CONSISTENCY.md @@ -97,7 +97,7 @@ Every error path returns a typed `RefusalCode`: * Capacity & Backpressure: `BackpressureOverCapacity`, `QuotaPayloadTooLarge`, `QuotaTooManyPaths`, `QuotaRateLimitExceeded`, `PaginationLimitExceeded` * Schema & Path Validation: `EnvelopeVersionUnsupported`, `PathInvalid`, `PathDuplicate`, `MutationsUnsorted`, `BaseTreeNotFound`, `ParentChangesetNotFound`, `ChangesetNotFound`, `VisibilitySeqAhead`, `IdempotencyKeyConflict` * Auth & Security: `AuthSignatureInvalid`, `AuthGrantSignatureInvalid`, `AuthGrantExpired`, `AuthCapabilityDenied`, `AuthCrossRepoDenied`, `AuthReplayDetected`, `AuthSelfReviewRefused` -* Git & Radicle Canonicality: `GitObjectCorrupt`, `GitRefNonFastForward`, `RadicleIdentityInvalid`, `RadicleSignatureInvalid`, `RadicleQuorumInsufficient`, `StaleBaseHead`, `SplitBrainDivergence` +* Git & Radicle Canonicality: `GitObjectCorrupt`, `GitObjectNotRepresentable`, `GitRefNonFastForward`, `RadicleIdentityInvalid`, `RadicleSignatureInvalid`, `RadicleQuorumInsufficient`, `StaleBaseHead`, `SplitBrainDivergence` ## 6. Schema and Storage Upgrades diff --git a/evidence/h1-git-bridge/SHA256SUMS b/evidence/h1-git-bridge/SHA256SUMS index b650bcb..2209f9c 100644 --- a/evidence/h1-git-bridge/SHA256SUMS +++ b/evidence/h1-git-bridge/SHA256SUMS @@ -1,4 +1,9 @@ +de58897161b1555e0ae0fc8c26e60c9dd0e993b8c2863450c85e3042e69066f9 red-green/cut2-green.txt +590a571d530f58ea1edbc22a1562820c5be367857b8ab19368f0d631bc86bc08 red-green/cut2-red.txt 8f8591c3edcf0019bd2d6e0fdc491cd33cf3c5e353990ec45bd171f6197ad60e red-green/green.txt 41088f6a7ba636697b9a239e9c37517eb9e442164e6f1f011f8e04649e1896f4 red-green/hostile-green.txt aaa6b740c5e1f28e2989e9e45dd50c2a97ce38d826426abf472a5cd7939297b7 red-green/hostile-red.txt 5daa9e456ef3ef01f695894887674b23d40c8d906fb379cefdd6c5746cc77a61 red-green/red.txt +14b0999e850603e021213540466ee72924bf8cf1987e6962b6c15bde89cf2638 red-green/rereview-red.txt +eb82aed6dba27d11966ce739dd1d1ab4f9d663a333b7b09fbe0d919be5b6db80 red-green/review96-green.txt +aa3b7e270722ad42be022aa89a0b7637586f4ca7245eb5e87691e21e9039f8c8 red-green/review96-red.txt diff --git a/evidence/h1-git-bridge/red-green/cut2-green.txt b/evidence/h1-git-bridge/red-green/cut2-green.txt new file mode 100644 index 0000000..1dba2fc --- /dev/null +++ b/evidence/h1-git-bridge/red-green/cut2-green.txt @@ -0,0 +1,49 @@ +# cut 2 green: cargo test --locked --workspace -j 4 (debug), feat/48-git-bridge-cut2 working tree +unittests src/lib.rs: test result: ok. 31 passed; 0 failed; 0 ignored +tests/b1_chained_revisions.rs: test result: ok. 2 passed; 0 failed; 0 ignored +tests/c1_tree_digest.rs: test result: ok. 1 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 1 passed; 0 failed; 0 ignored +tests/f1_capability_chains.rs: test result: ok. 11 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 74 passed; 0 failed; 0 ignored +unittests src/bin/verify-qualification.rs: test result: ok. 6 passed; 0 failed; 0 ignored +tests/c1_tree_cache_bound.rs: test result: ok. 2 passed; 0 failed; 0 ignored +tests/c1_tree_parity.rs: test result: ok. 2 passed; 0 failed; 1 ignored +tests/crumb_synapse_qualification.rs: test result: ok. 15 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 9 passed; 0 failed; 0 ignored +tests/hostile.rs: test result: ok. 11 passed; 0 failed; 0 ignored +tests/identity.rs: test result: ok. 9 passed; 0 failed; 0 ignored +tests/plane.rs: test result: ok. 12 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 14 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 16 passed; 0 failed; 0 ignored +tests/parity.rs: test result: ok. 8 passed; 0 failed; 0 ignored +tests/repo_root.rs: test result: ok. 1 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 15 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 16 passed; 0 failed; 0 ignored +tests/h1_bridge_import.rs: test result: ok. 8 passed; 0 failed; 1 ignored +tests/h1_git_bridge.rs: test result: ok. 6 passed; 0 failed; 1 ignored +tests/h1_hostile_repo.rs: test result: ok. 9 passed; 0 failed; 0 ignored +tests/h1_read_limits.rs: test result: ok. 4 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 2 passed; 0 failed; 0 ignored +tests/e0_identity_document.rs: test result: ok. 6 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: ok. 55 passed; 0 failed; 0 ignored +tests/a1_malformed_corpus.rs: test result: ok. 2 passed; 0 failed; 0 ignored +tests/b1_change_identity.rs: test result: ok. 8 passed; 0 failed; 0 ignored +tests/g1_fuzz.rs: test result: ok. 7 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: ok. 8 passed; 0 failed; 0 ignored +workspace total: passed 371 failed 0 ignored 3 + +# real repositories through the store (release test binary, /usr/bin/time -v, mirror clones) +== synapse (release test binary, main 3d78efe) +real repository /synapse.git through the store: ImportReceipt { blobs: 876, trees: 859, commits: 180, tags: 0, refs: 75, records_written: 183, records_skipped: 0 }; import 7.5 s, reopen (WAL replay) 0.1 s, export 1.0 s, pack 72477923 bytes +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 8 filtered out; finished in 8.94s + Elapsed (wall clock) time (h:mm:ss or m:ss): 0:08.94 + Maximum resident set size (kbytes): 197116 +== omega (release test binary, main f4a9849f) +real repository /omega.git through the store: ImportReceipt { blobs: 16867, trees: 10345, commits: 2413, tags: 11, refs: 633, records_written: 2433, records_skipped: 0 }; import 123.3 s, reopen (WAL replay) 0.3 s, export 5.8 s, pack 374713444 bytes +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 8 filtered out; finished in 131.72s + Elapsed (wall clock) time (h:mm:ss or m:ss): 2:11.72 + Maximum resident set size (kbytes): 976824 + +# lint +cargo clippy --locked --workspace --all-targets -j 4 -- -D warnings: clean +cargo fmt --all --check: clean diff --git a/evidence/h1-git-bridge/red-green/cut2-red.txt b/evidence/h1-git-bridge/red-green/cut2-red.txt new file mode 100644 index 0000000..e2e0d92 --- /dev/null +++ b/evidence/h1-git-bridge/red-green/cut2-red.txt @@ -0,0 +1,35 @@ +# cut 2 red: each stub removes one behaviour; the named test fails by assertion (stub applied, test run, file restored) +== S1 canonical-form check skipped (legacy/padded modes accepted) (test h1_import_refuses_trees_with_no_native_form_before_writing) +thread 'h1_import_refuses_trees_with_no_native_form_before_writing' (2687664) panicked at crates/synapse-git/tests/h1_bridge_import.rs:418:18: +legacy mode 100664: imported instead of refusing: ImportReceipt { blobs: 1, trees: 1, commits: 1, tags: 0, refs: 1, records_written: 3, records_skipped: 0 } +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.05s +== S2 commit headers dropped on import (test h1_import_then_export_from_the_store_gives_identical_ids) +thread 'h1_import_then_export_from_the_store_gives_identical_ids_refs_and_fsck_strict_clean' (2687960) panicked at crates/synapse-git/tests/h1_bridge_import.rs:62:29: +export refused: GitObjectCorrupt: stored commit rebuilds as 13a42cd42dc51afe60fa58c32e89be5093e7548e, not its imported Git ID 13912c94cf6dd0b30ba74d7ec5c8bc88231b7c77 +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 1.52s +== S3 bridge map not rebuilt on checkpoint load (test h1_bridge_map_survives_reopen_and_checkpoint) +thread 'h1_bridge_map_survives_reopen_and_checkpoint_with_the_same_export' (2688999) panicked at crates/synapse-git/tests/h1_bridge_import.rs:154:5: +assertion `left == right` failed: a checkpoint load rebuilds the same map + left: GitOid(029d08823bd8a8eab510ad6ac75c823cfd3ed31e) + right: GitOid(110cae2138cf180d22011dd488d3140f543a60a2) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 3.50s +== S4 native commit time taken from the clock (test h1_native_revisions_export_deterministically) +thread 'h1_native_revisions_export_deterministically_on_imported_history' (2690977) panicked at crates/synapse-git/tests/h1_bridge_import.rs:320:5: +assertion `left == right` failed: two exports give the same pack + left: GitOid(6550ec70f1840e943c4439e8e36bb31e275496ea) + right: GitOid(1e7e30cc594b42b47ad46b5ebd0fabe4f4df02f6) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 2.83s +== S5 reference names checked only by the store (after objects are written) (test h1_import_refuses_reference_names) +thread 'h1_import_refuses_reference_names_that_would_write_outside_before_writing' (2692013) panicked at crates/synapse-git/tests/h1_bridge_import.rs:420:5: +assertion `left == right` failed: dot-dot path: nothing written + left: WalSeq(2) + right: WalSeq(0) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.04s +== S6 compaction drops bridge-held blobs (test h1_bridge_map_survives_reopen_and_checkpoint) +thread 'h1_bridge_map_survives_reopen_and_checkpoint_with_the_same_export' (2693564) panicked at crates/synapse-git/tests/h1_bridge_import.rs:62:29: +export refused: BlobNotFound: blob bec6e547ba97d2d4cb34f2a3de81e1110522735fe04188a1eb9ae7330ca23421 is not committed in the WAL +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 2.99s +== S7 native result trees not remembered (a native child's base is unknown) (test h1_a_native_child_of_a_native_revision) +thread 'h1_a_native_child_of_a_native_revision_exports_on_its_parents_result_tree' (2695099) panicked at crates/synapse-git/tests/h1_bridge_import.rs:62:29: +export refused: TreeNodeNotFound: base tree node 9fa91d0179144debfd7d53724f9dff9aef0051e36363d5f40650d13106281885 is not stored in the bridge +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 3.40s diff --git a/evidence/h1-git-bridge/red-green/rereview-red.txt b/evidence/h1-git-bridge/red-green/rereview-red.txt new file mode 100644 index 0000000..f766ff2 --- /dev/null +++ b/evidence/h1-git-bridge/red-green/rereview-red.txt @@ -0,0 +1,10 @@ +# Re-review N1/N3 red run: 28d783c + working tree, one stub at a time +## stub N1: fan-out directory check removed (if !real_dir(&fanout)? -> if false) +test h1_hostile_linked_object_fanout_directory_is_refused ... FAILED +thread 'h1_hostile_linked_object_fanout_directory_is_refused' (2242811) panicked at crates/synapse-git/tests/h1_hostile_repo.rs:80:18: +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 8 filtered out; finished in 0.02s +## stub N3: indexer total budget check removed (declared > max_total_object_bytes -> false) +test pack::tests::indexer_resolves_a_reference_delta_stored_before_its_base_within_the_budget ... FAILED +thread 'pack::tests::indexer_resolves_a_reference_delta_stored_before_its_base_within_the_budget' (2249045) panicked at crates/synapse-git/src/pack.rs:771:22: +one byte short: indexed 2 entries instead of refusing +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 15 filtered out; finished in 0.00s diff --git a/evidence/h1-git-bridge/red-green/review96-green.txt b/evidence/h1-git-bridge/red-green/review96-green.txt new file mode 100644 index 0000000..d540d10 --- /dev/null +++ b/evidence/h1-git-bridge/red-green/review96-green.txt @@ -0,0 +1,71 @@ +# review of #96 green: cargo test --locked --workspace -j 4 (debug), feat/48-git-bridge-cut2 working tree +unittests src/lib.rs: test result: 31 passed; 0 failed; 0 ignored +tests/b1_chained_revisions.rs: test result: 2 passed; 0 failed; 0 ignored +tests/c1_tree_digest.rs: test result: 1 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 1 passed; 0 failed; 0 ignored +tests/f1_capability_chains.rs: test result: 11 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 74 passed; 0 failed; 0 ignored +unittests src/main.rs: test result: 0 passed; 0 failed; 0 ignored +unittests src/bin/verify-qualification.rs: test result: 6 passed; 0 failed; 0 ignored +unittests src/bin/wire-transport.rs: test result: 0 passed; 0 failed; 0 ignored +tests/c1_tree_cache_bound.rs: test result: 2 passed; 0 failed; 0 ignored +tests/c1_tree_parity.rs: test result: 2 passed; 0 failed; 1 ignored +tests/crumb_synapse_qualification.rs: test result: 15 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 9 passed; 0 failed; 0 ignored +unittests src/bin/synapse-coord.rs: test result: 0 passed; 0 failed; 0 ignored +tests/hostile.rs: test result: 11 passed; 0 failed; 0 ignored +tests/identity.rs: test result: 9 passed; 0 failed; 0 ignored +tests/plane.rs: test result: 12 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 14 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 16 passed; 0 failed; 0 ignored +tests/parity.rs: test result: 8 passed; 0 failed; 0 ignored +tests/repo_root.rs: test result: 1 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 15 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 17 passed; 0 failed; 0 ignored +tests/h1_bridge_import.rs: test result: 10 passed; 0 failed; 1 ignored +tests/h1_git_bridge.rs: test result: 6 passed; 0 failed; 1 ignored +tests/h1_hostile_repo.rs: test result: 9 passed; 0 failed; 0 ignored +tests/h1_read_limits.rs: test result: 4 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 2 passed; 0 failed; 0 ignored +tests/e0_identity_document.rs: test result: 6 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 55 passed; 0 failed; 0 ignored +tests/a1_malformed_corpus.rs: test result: 2 passed; 0 failed; 0 ignored +tests/b1_change_identity.rs: test result: 8 passed; 0 failed; 0 ignored +unittests src/lib.rs: test result: 0 passed; 0 failed; 0 ignored +unittests src/bin/synapse-wire-fuzz.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_fuzz.rs: test result: 7 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 8 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored +tests/g1_wire.rs: test result: 0 passed; 0 failed; 0 ignored + +# the tests added for the review +test bridge::tests::export_refuses_a_tag_target_that_is_not_imported ... ok +test h1_import_refuses_a_record_over_the_store_limits_before_writing ... ok +test h1_store_refuses_git_ids_that_do_not_match_their_content ... ok + +total: 374 passed, 0 failed, 3 ignored + +# real repositories through the store (release test binary, /usr/bin/time -v, same mirror clones as cut2-green.txt; host load average about 14.5 on 20 cores during the run, so times are not comparable to cut2-green.txt) +== synapse +real repository /synapse.git through the store: ImportReceipt { blobs: 876, trees: 859, commits: 180, tags: 0, refs: 75, records_written: 183, records_skipped: 0 }; import 8.1 s, reopen (WAL replay) 0.1 s, export 0.9 s, pack 72477923 bytes +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 10 filtered out; finished in 9.48s + Elapsed (wall clock) time (h:mm:ss or m:ss): 0:09.47 + Maximum resident set size (kbytes): 197056 +== omega +real repository /omega.git through the store: ImportReceipt { blobs: 16867, trees: 10345, commits: 2413, tags: 11, refs: 633, records_written: 2433, records_skipped: 0 }; import 133.7 s, reopen (WAL replay) 4.1 s, export 15.1 s, pack 374713444 bytes +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 10 filtered out; finished in 155.23s + Elapsed (wall clock) time (h:mm:ss or m:ss): 2:35.23 + Maximum resident set size (kbytes): 977812 + +# lint +cargo clippy --locked --workspace --all-targets -j 4 -- -D warnings: clean +cargo fmt --all --check: clean diff --git a/evidence/h1-git-bridge/red-green/review96-red.txt b/evidence/h1-git-bridge/red-green/review96-red.txt new file mode 100644 index 0000000..3b9e9e4 --- /dev/null +++ b/evidence/h1-git-bridge/red-green/review96-red.txt @@ -0,0 +1,46 @@ +== Baseline: the new tests on b69db2d (the reviewed head), unchanged code +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3328942) panicked at crates/synapse-git/tests/h1_bridge_import.rs:777:22: +one Git ID mapped to two blobs in one entry: accepted (Some(WalSeq(1))) +thread 'h1_import_refuses_a_record_over_the_store_limits_before_writing' (3328932) panicked at crates/synapse-git/tests/h1_bridge_import.rs:420:5: +assertion `left == right` failed: commit record over max_wal_record_bytes: nothing written + left: WalSeq(1) + right: WalSeq(0) +test result: FAILED. 8 passed; 2 failed; 1 ignored; 0 measured; 0 filtered out; finished in 4.74s + +== R1 dry run skips the WAL frame size (B1) (test h1_import_refuses_a_record_over_the_store_limits) +thread 'h1_import_refuses_a_record_over_the_store_limits_before_writing' (3747395) panicked at crates/synapse-git/tests/h1_bridge_import.rs:420:5: +assertion `left == right` failed: commit record over max_wal_record_bytes: nothing written + left: WalSeq(1) + right: WalSeq(0) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.91s +== R2 importer writes record by record, no whole-import dry run (B1) (test h1_import_refuses_a_record_over_the_store_limits) +thread 'h1_import_refuses_a_record_over_the_store_limits_before_writing' (3748760) panicked at crates/synapse-git/tests/h1_bridge_import.rs:420:5: +assertion `left == right` failed: commit record over max_wal_record_bytes: nothing written + left: WalSeq(1) + right: WalSeq(0) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.06s +== R3 blob Git ID not re-hashed (B2) (test h1_store_refuses_git_ids_that_do_not_match) +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3749726) panicked at crates/synapse-git/tests/h1_bridge_import.rs:777:22: +a blob under another blob's Git ID: accepted (Some(WalSeq(1))) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.03s +== R4 tree Git ID not re-hashed (B2) (test h1_store_refuses_git_ids_that_do_not_match) +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3750712) panicked at crates/synapse-git/tests/h1_bridge_import.rs:777:22: +a tree under a wrong Git ID: accepted (Some(WalSeq(1))) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.05s +== R5 commit Git ID not re-hashed (B2) (test h1_store_refuses_git_ids_that_do_not_match) +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3752073) panicked at crates/synapse-git/tests/h1_bridge_import.rs:871:22: +a commit under a wrong Git ID: accepted (Some(WalSeq(2))) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.03s +== R6 tag Git ID not re-hashed (B2) (test h1_store_refuses_git_ids_that_do_not_match) +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3752894) panicked at crates/synapse-git/tests/h1_bridge_import.rs:905:18: +a tag under a wrong Git ID: accepted (Some(WalSeq(3))) +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.04s +== R7 tree file blob need only be visible, not mapped in its repository (N3) (test h1_store_refuses_git_ids_that_do_not_match) +thread 'h1_store_refuses_git_ids_that_do_not_match_their_content' (3753679) panicked at crates/synapse-git/tests/h1_bridge_import.rs:851:19: +assertion `left == right` failed: GitObjectCorrupt: unmapped entry + left: GitObjectCorrupt + right: BlobNotFound +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 10 filtered out; finished in 0.03s +== R8 export passes any tag Git ID through as a target (test export_refuses_a_tag_target_that_is_not_imported) +thread 'bridge::tests::export_refuses_a_tag_target_that_is_not_imported' (3754688) panicked at crates/synapse-git/src/bridge.rs:887:58: +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 16 filtered out; finished in 0.00s diff --git a/evidence/j1-codec-corpus/corpus.bin b/evidence/j1-codec-corpus/corpus.bin index 52993d0..7343779 100644 Binary files a/evidence/j1-codec-corpus/corpus.bin and b/evidence/j1-codec-corpus/corpus.bin differ